From 930698ac0e8d92bcb535675e247d57aed1dc62ab Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 19 Feb 2026 16:25:42 +1100 Subject: [PATCH 01/15] Replace Playwright MCP server with Playwright CLI installation Replace the Playwright MCP server configuration in `aspire agent init` with a secure Playwright CLI installation workflow. Instead of writing MCP server configuration to each agent environment's config file, the new approach: - Resolves the @playwright/cli package version from npm registry - Downloads the package tarball via `npm pack` - Verifies supply chain integrity (SHA-512 SRI hash comparison) - Runs `npm audit signatures` for provenance verification - Installs globally from the verified tarball - Runs `playwright-cli install --skills` to generate skill files New abstractions: - INpmRunner/NpmRunner: npm CLI command runner (resolve, pack, audit, install) - IPlaywrightCliRunner/PlaywrightCliRunner: playwright-cli command runner - PlaywrightCliInstaller: orchestrates the secure install flow This removes ~400 lines of per-scanner MCP config writing code (different JSON formats for VS Code, Claude Code, Copilot CLI, and OpenCode) and replaces it with a single global CLI install. Fixes #14430 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Agents/AgentEnvironmentScanContext.cs | 22 +- .../ClaudeCodeAgentEnvironmentScanner.cs | 100 ++---- .../Agents/CommonAgentApplicators.cs | 31 +- .../CopilotCliAgentEnvironmentScanner.cs | 112 ++---- .../OpenCodeAgentEnvironmentScanner.cs | 102 +++--- .../Agents/Playwright/IPlaywrightCliRunner.cs | 26 ++ .../Playwright/PlaywrightCliInstaller.cs | 154 ++++++++ .../Agents/Playwright/PlaywrightCliRunner.cs | 128 +++++++ .../VsCode/VsCodeAgentEnvironmentScanner.cs | 99 ++--- src/Aspire.Cli/Npm/INpmRunner.cs | 62 ++++ src/Aspire.Cli/Npm/NpmRunner.cs | 186 ++++++++++ src/Aspire.Cli/Program.cs | 5 + .../Resources/McpCommandStrings.Designer.cs | 2 +- .../Resources/McpCommandStrings.resx | 2 +- .../Resources/xlf/McpCommandStrings.cs.xlf | 4 +- .../Resources/xlf/McpCommandStrings.de.xlf | 4 +- .../Resources/xlf/McpCommandStrings.es.xlf | 4 +- .../Resources/xlf/McpCommandStrings.fr.xlf | 4 +- .../Resources/xlf/McpCommandStrings.it.xlf | 4 +- .../Resources/xlf/McpCommandStrings.ja.xlf | 4 +- .../Resources/xlf/McpCommandStrings.ko.xlf | 4 +- .../Resources/xlf/McpCommandStrings.pl.xlf | 4 +- .../Resources/xlf/McpCommandStrings.pt-BR.xlf | 4 +- .../Resources/xlf/McpCommandStrings.ru.xlf | 4 +- .../Resources/xlf/McpCommandStrings.tr.xlf | 4 +- .../xlf/McpCommandStrings.zh-Hans.xlf | 4 +- .../xlf/McpCommandStrings.zh-Hant.xlf | 4 +- .../ClaudeCodeAgentEnvironmentScannerTests.cs | 16 +- .../CopilotCliAgentEnvironmentScannerTests.cs | 43 ++- .../OpenCodeAgentEnvironmentScannerTests.cs | 16 +- .../Agents/PlaywrightCliInstallerTests.cs | 338 ++++++++++++++++++ .../VsCodeAgentEnvironmentScannerTests.cs | 72 ++-- .../TestServices/FakePlaywrightServices.cs | 38 ++ 33 files changed, 1213 insertions(+), 393 deletions(-) create mode 100644 src/Aspire.Cli/Agents/Playwright/IPlaywrightCliRunner.cs create mode 100644 src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs create mode 100644 src/Aspire.Cli/Agents/Playwright/PlaywrightCliRunner.cs create mode 100644 src/Aspire.Cli/Npm/INpmRunner.cs create mode 100644 src/Aspire.Cli/Npm/NpmRunner.cs create mode 100644 tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs create mode 100644 tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs diff --git a/src/Aspire.Cli/Agents/AgentEnvironmentScanContext.cs b/src/Aspire.Cli/Agents/AgentEnvironmentScanContext.cs index 9794e7e6807..8d43c58f7e1 100644 --- a/src/Aspire.Cli/Agents/AgentEnvironmentScanContext.cs +++ b/src/Aspire.Cli/Agents/AgentEnvironmentScanContext.cs @@ -24,31 +24,11 @@ internal sealed class AgentEnvironmentScanContext public required DirectoryInfo RepositoryRoot { get; init; } /// - /// Gets or sets a value indicating whether a Playwright applicator has been added. + /// Gets or sets a value indicating whether a Playwright CLI applicator has been added. /// This is used to ensure only one applicator for Playwright is added across all scanners. /// public bool PlaywrightApplicatorAdded { get; set; } - /// - /// Stores the Playwright configuration callbacks from each scanner. - /// These will be executed if the user selects to configure Playwright. - /// - private readonly List> _playwrightConfigurationCallbacks = []; - - /// - /// Adds a Playwright configuration callback for a specific environment. - /// - /// The callback to execute if Playwright is configured. - public void AddPlaywrightConfigurationCallback(Func callback) - { - _playwrightConfigurationCallbacks.Add(callback); - } - - /// - /// Gets all registered Playwright configuration callbacks. - /// - public IReadOnlyList> PlaywrightConfigurationCallbacks => _playwrightConfigurationCallbacks; - /// /// Checks if a skill file applicator has already been added for the specified path. /// diff --git a/src/Aspire.Cli/Agents/ClaudeCode/ClaudeCodeAgentEnvironmentScanner.cs b/src/Aspire.Cli/Agents/ClaudeCode/ClaudeCodeAgentEnvironmentScanner.cs index 1b613bd6b25..a4e225a0c2a 100644 --- a/src/Aspire.Cli/Agents/ClaudeCode/ClaudeCodeAgentEnvironmentScanner.cs +++ b/src/Aspire.Cli/Agents/ClaudeCode/ClaudeCodeAgentEnvironmentScanner.cs @@ -3,6 +3,7 @@ using System.Text.Json; using System.Text.Json.Nodes; +using Aspire.Cli.Agents.Playwright; using Aspire.Cli.Resources; using Microsoft.Extensions.Logging; @@ -20,6 +21,7 @@ internal sealed class ClaudeCodeAgentEnvironmentScanner : IAgentEnvironmentScann private const string SkillFileDescription = "Create Aspire skill file (.claude/skills/aspire/SKILL.md)"; private readonly IClaudeCodeCliRunner _claudeCodeCliRunner; + private readonly PlaywrightCliInstaller _playwrightCliInstaller; private readonly CliExecutionContext _executionContext; private readonly ILogger _logger; @@ -27,14 +29,17 @@ internal sealed class ClaudeCodeAgentEnvironmentScanner : IAgentEnvironmentScann /// Initializes a new instance of . /// /// The Claude Code CLI runner for checking if Claude Code is installed. + /// The Playwright CLI installer for secure installation. /// The CLI execution context for accessing environment variables and settings. /// The logger for diagnostic output. - public ClaudeCodeAgentEnvironmentScanner(IClaudeCodeCliRunner claudeCodeCliRunner, CliExecutionContext executionContext, ILogger logger) + public ClaudeCodeAgentEnvironmentScanner(IClaudeCodeCliRunner claudeCodeCliRunner, PlaywrightCliInstaller playwrightCliInstaller, CliExecutionContext executionContext, ILogger logger) { ArgumentNullException.ThrowIfNull(claudeCodeCliRunner); + ArgumentNullException.ThrowIfNull(playwrightCliInstaller); ArgumentNullException.ThrowIfNull(executionContext); ArgumentNullException.ThrowIfNull(logger); _claudeCodeCliRunner = claudeCodeCliRunner; + _playwrightCliInstaller = playwrightCliInstaller; _executionContext = executionContext; _logger = logger; } @@ -68,18 +73,8 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok _logger.LogDebug("Aspire MCP server is already configured"); } - // Register Playwright configuration callback if not already configured - if (!HasPlaywrightServerConfigured(workspaceRoot)) - { - _logger.LogDebug("Registering Playwright MCP configuration callback for Claude Code"); - CommonAgentApplicators.AddPlaywrightConfigurationCallback( - context, - ct => ApplyPlaywrightMcpConfigurationAsync(workspaceRoot, ct)); - } - else - { - _logger.LogDebug("Playwright MCP server is already configured"); - } + // Register Playwright CLI installation applicator + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); // Try to add skill file applicator for Claude Code CommonAgentApplicators.TryAddSkillFileApplicator( @@ -109,18 +104,8 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok _logger.LogDebug("Aspire MCP server is already configured"); } - // Register Playwright configuration callback if not already configured - if (!HasPlaywrightServerConfigured(context.RepositoryRoot)) - { - _logger.LogDebug("Registering Playwright MCP configuration callback for Claude Code"); - CommonAgentApplicators.AddPlaywrightConfigurationCallback( - context, - ct => ApplyPlaywrightMcpConfigurationAsync(context.RepositoryRoot, ct)); - } - else - { - _logger.LogDebug("Playwright MCP server is already configured"); - } + // Register Playwright CLI installation applicator + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); // Try to add skill file applicator for Claude Code CommonAgentApplicators.TryAddSkillFileApplicator( @@ -179,16 +164,34 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok private static bool HasAspireServerConfigured(DirectoryInfo repoRoot) { var configFilePath = Path.Combine(repoRoot.FullName, McpConfigFileName); - return McpConfigFileHelper.HasServerConfigured(configFilePath, "mcpServers", AspireServerName); - } - /// - /// Checks if the Playwright MCP server is already configured in the .mcp.json file. - /// - private static bool HasPlaywrightServerConfigured(DirectoryInfo repoRoot) - { - var configFilePath = Path.Combine(repoRoot.FullName, McpConfigFileName); - return McpConfigFileHelper.HasServerConfigured(configFilePath, "mcpServers", "playwright"); + if (!File.Exists(configFilePath)) + { + return false; + } + + try + { + var content = File.ReadAllText(configFilePath); + var config = JsonNode.Parse(content)?.AsObject(); + + if (config is null) + { + return false; + } + + if (config.TryGetPropertyValue("mcpServers", out var serversNode) && serversNode is JsonObject servers) + { + return servers.ContainsKey(AspireServerName); + } + + return false; + } + catch (JsonException) + { + // If the JSON is malformed, assume aspire is not configured + return false; + } } /// @@ -231,33 +234,4 @@ private static async Task ApplyAspireMcpConfigurationAsync( await File.WriteAllTextAsync(configFilePath, jsonContent, cancellationToken); } - /// - /// Creates or updates the .mcp.json file at the repo root with Playwright MCP configuration. - /// - private static async Task ApplyPlaywrightMcpConfigurationAsync( - DirectoryInfo repoRoot, - CancellationToken cancellationToken) - { - var configFilePath = Path.Combine(repoRoot.FullName, McpConfigFileName); - var config = await McpConfigFileHelper.ReadConfigAsync(configFilePath, cancellationToken); - - // Ensure "mcpServers" object exists - if (!config.ContainsKey("mcpServers") || config["mcpServers"] is not JsonObject) - { - config["mcpServers"] = new JsonObject(); - } - - var servers = config["mcpServers"]!.AsObject(); - - // Add Playwright MCP server configuration - servers["playwright"] = new JsonObject - { - ["command"] = "npx", - ["args"] = new JsonArray("-y", "@playwright/mcp@latest") - }; - - // Write the updated config using AOT-compatible serialization - var jsonContent = JsonSerializer.Serialize(config, JsonSourceGenerationContext.Default.JsonObject); - await File.WriteAllTextAsync(configFilePath, jsonContent, cancellationToken); - } } diff --git a/src/Aspire.Cli/Agents/CommonAgentApplicators.cs b/src/Aspire.Cli/Agents/CommonAgentApplicators.cs index 271c5db22c8..e2b8d070342 100644 --- a/src/Aspire.Cli/Agents/CommonAgentApplicators.cs +++ b/src/Aspire.Cli/Agents/CommonAgentApplicators.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Aspire.Cli.Agents.Playwright; + namespace Aspire.Cli.Agents; /// @@ -73,18 +75,16 @@ public static bool TryAddSkillFileApplicator( } /// - /// Tracks a detected environment and adds a single Playwright applicator if not already added. - /// This should be called by each scanner that detects an environment supporting Playwright. + /// Adds a single Playwright CLI installation applicator if not already added. + /// Called by scanners that detect an environment supporting Playwright. + /// The applicator uses to securely install the CLI and generate skill files. /// /// The scan context. - /// The callback to configure Playwright for this specific environment. - public static void AddPlaywrightConfigurationCallback( + /// The Playwright CLI installer that handles secure installation. + public static void AddPlaywrightCliApplicator( AgentEnvironmentScanContext context, - Func configurationCallback) + PlaywrightCliInstaller installer) { - // Add this environment's Playwright configuration callback - context.AddPlaywrightConfigurationCallback(configurationCallback); - // Only add the Playwright applicator prompt once across all environments if (context.PlaywrightApplicatorAdded) { @@ -93,15 +93,8 @@ public static void AddPlaywrightConfigurationCallback( context.PlaywrightApplicatorAdded = true; context.AddApplicator(new AgentEnvironmentApplicator( - "Configure Playwright MCP server", - async ct => - { - // Execute all registered Playwright configuration callbacks - foreach (var callback in context.PlaywrightConfigurationCallbacks) - { - await callback(ct); - } - }, + "Install Playwright CLI for browser automation", + installer.InstallAsync, promptGroup: McpInitPromptGroup.AdditionalOptions, priority: 1)); } @@ -235,9 +228,9 @@ aspire run 1. _select apphost_; use this tool if working with multiple app hosts within a workspace. 2. _list apphosts_; use this tool to get details about active app hosts. - ## Playwright MCP server + ## Playwright CLI - The playwright MCP server has also been configured in this repository and you should use it to perform functional investigations of the resources defined in the app model as you work on the codebase. To get endpoints that can be used for navigation using the playwright MCP server use the list resources tool. + The Playwright CLI has been installed in this repository for browser automation. Use it to perform functional investigations of the resources defined in the app model as you work on the codebase. To get endpoints that can be used for navigation use the list resources tool. Run `playwright-cli --help` for available commands. ## Updating the app host diff --git a/src/Aspire.Cli/Agents/CopilotCli/CopilotCliAgentEnvironmentScanner.cs b/src/Aspire.Cli/Agents/CopilotCli/CopilotCliAgentEnvironmentScanner.cs index cb40cf3da30..40d53114b4f 100644 --- a/src/Aspire.Cli/Agents/CopilotCli/CopilotCliAgentEnvironmentScanner.cs +++ b/src/Aspire.Cli/Agents/CopilotCli/CopilotCliAgentEnvironmentScanner.cs @@ -3,6 +3,7 @@ using System.Text.Json; using System.Text.Json.Nodes; +using Aspire.Cli.Agents.Playwright; using Aspire.Cli.Resources; using Microsoft.Extensions.Logging; @@ -20,6 +21,7 @@ internal sealed class CopilotCliAgentEnvironmentScanner : IAgentEnvironmentScann private const string SkillFileDescription = "Create Aspire skill file (.github/skills/aspire/SKILL.md)"; private readonly ICopilotCliRunner _copilotCliRunner; + private readonly PlaywrightCliInstaller _playwrightCliInstaller; private readonly CliExecutionContext _executionContext; private readonly ILogger _logger; @@ -27,14 +29,17 @@ internal sealed class CopilotCliAgentEnvironmentScanner : IAgentEnvironmentScann /// Initializes a new instance of . /// /// The Copilot CLI runner for checking if Copilot CLI is installed. + /// The Playwright CLI installer for secure installation. /// The CLI execution context for accessing environment variables and settings. /// The logger for diagnostic output. - public CopilotCliAgentEnvironmentScanner(ICopilotCliRunner copilotCliRunner, CliExecutionContext executionContext, ILogger logger) + public CopilotCliAgentEnvironmentScanner(ICopilotCliRunner copilotCliRunner, PlaywrightCliInstaller playwrightCliInstaller, CliExecutionContext executionContext, ILogger logger) { ArgumentNullException.ThrowIfNull(copilotCliRunner); + ArgumentNullException.ThrowIfNull(playwrightCliInstaller); ArgumentNullException.ThrowIfNull(executionContext); ArgumentNullException.ThrowIfNull(logger); _copilotCliRunner = copilotCliRunner; + _playwrightCliInstaller = playwrightCliInstaller; _executionContext = executionContext; _logger = logger; } @@ -67,18 +72,8 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok _logger.LogDebug("Aspire MCP server is already configured in Copilot CLI"); } - // Register Playwright configuration callback if not already configured - if (!HasPlaywrightServerConfigured(homeDirectory)) - { - _logger.LogDebug("Registering Playwright MCP configuration callback for Copilot CLI"); - CommonAgentApplicators.AddPlaywrightConfigurationCallback( - context, - ct => ApplyPlaywrightMcpConfigurationAsync(homeDirectory, ct)); - } - else - { - _logger.LogDebug("Playwright MCP server is already configured in Copilot CLI"); - } + // Register Playwright CLI installation applicator + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); // Try to add skill file applicator for GitHub Copilot CommonAgentApplicators.TryAddSkillFileApplicator( @@ -115,18 +110,8 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok _logger.LogDebug("Aspire MCP server is already configured in Copilot CLI"); } - // Register Playwright configuration callback if not already configured - if (!HasPlaywrightServerConfigured(homeDirectory)) - { - _logger.LogDebug("Registering Playwright MCP configuration callback for Copilot CLI"); - CommonAgentApplicators.AddPlaywrightConfigurationCallback( - context, - ct => ApplyPlaywrightMcpConfigurationAsync(homeDirectory, ct)); - } - else - { - _logger.LogDebug("Playwright MCP server is already configured in Copilot CLI"); - } + // Register Playwright CLI installation applicator + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); // Try to add skill file applicator for GitHub Copilot CommonAgentApplicators.TryAddSkillFileApplicator( @@ -162,7 +147,34 @@ private static string GetMcpConfigFilePath(DirectoryInfo homeDirectory) private static bool HasAspireServerConfigured(DirectoryInfo homeDirectory) { var configFilePath = GetMcpConfigFilePath(homeDirectory); - return McpConfigFileHelper.HasServerConfigured(configFilePath, "mcpServers", AspireServerName); + + if (!File.Exists(configFilePath)) + { + return false; + } + + try + { + var content = File.ReadAllText(configFilePath); + var config = JsonNode.Parse(content)?.AsObject(); + + if (config is null) + { + return false; + } + + if (config.TryGetPropertyValue("mcpServers", out var serversNode) && serversNode is JsonObject servers) + { + return servers.ContainsKey(AspireServerName); + } + + return false; + } + catch (JsonException) + { + // If the JSON is malformed, assume aspire is not configured + return false; + } } /// @@ -224,52 +236,4 @@ private static async Task ApplyMcpConfigurationAsync( await File.WriteAllTextAsync(configFilePath, jsonContent, cancellationToken); } - /// - /// Creates or updates the mcp-config.json file with Playwright MCP configuration. - /// - private static async Task ApplyPlaywrightMcpConfigurationAsync( - DirectoryInfo homeDirectory, - CancellationToken cancellationToken) - { - var configDirectory = GetCopilotConfigDirectory(homeDirectory); - var configFilePath = GetMcpConfigFilePath(homeDirectory); - - // Ensure the .copilot directory exists - if (!Directory.Exists(configDirectory)) - { - Directory.CreateDirectory(configDirectory); - } - - var config = await McpConfigFileHelper.ReadConfigAsync(configFilePath, cancellationToken); - - // Ensure "mcpServers" object exists - if (!config.ContainsKey("mcpServers") || config["mcpServers"] is not JsonObject) - { - config["mcpServers"] = new JsonObject(); - } - - var servers = config["mcpServers"]!.AsObject(); - - // Add Playwright MCP server configuration - servers["playwright"] = new JsonObject - { - ["type"] = "local", - ["command"] = "npx", - ["args"] = new JsonArray("-y", "@playwright/mcp@latest"), - ["tools"] = new JsonArray("*") - }; - - // Write the updated config using AOT-compatible serialization - var jsonContent = JsonSerializer.Serialize(config, JsonSourceGenerationContext.Default.JsonObject); - await File.WriteAllTextAsync(configFilePath, jsonContent, cancellationToken); - } - - /// - /// Checks if the Playwright MCP server is already configured in the mcp-config.json file. - /// - private static bool HasPlaywrightServerConfigured(DirectoryInfo homeDirectory) - { - var configFilePath = GetMcpConfigFilePath(homeDirectory); - return McpConfigFileHelper.HasServerConfigured(configFilePath, "mcpServers", "playwright"); - } } diff --git a/src/Aspire.Cli/Agents/OpenCode/OpenCodeAgentEnvironmentScanner.cs b/src/Aspire.Cli/Agents/OpenCode/OpenCodeAgentEnvironmentScanner.cs index 1674a690e29..d2036b16470 100644 --- a/src/Aspire.Cli/Agents/OpenCode/OpenCodeAgentEnvironmentScanner.cs +++ b/src/Aspire.Cli/Agents/OpenCode/OpenCodeAgentEnvironmentScanner.cs @@ -3,6 +3,7 @@ using System.Text.Json; using System.Text.Json.Nodes; +using Aspire.Cli.Agents.Playwright; using Aspire.Cli.Resources; using Microsoft.Extensions.Logging; @@ -19,18 +20,22 @@ internal sealed class OpenCodeAgentEnvironmentScanner : IAgentEnvironmentScanner private const string SkillFileDescription = "Create Aspire skill file (.opencode/skill/aspire/SKILL.md)"; private readonly IOpenCodeCliRunner _openCodeCliRunner; + private readonly PlaywrightCliInstaller _playwrightCliInstaller; private readonly ILogger _logger; /// /// Initializes a new instance of . /// /// The OpenCode CLI runner for checking if OpenCode is installed. + /// The Playwright CLI installer for secure installation. /// The logger for diagnostic output. - public OpenCodeAgentEnvironmentScanner(IOpenCodeCliRunner openCodeCliRunner, ILogger logger) + public OpenCodeAgentEnvironmentScanner(IOpenCodeCliRunner openCodeCliRunner, PlaywrightCliInstaller playwrightCliInstaller, ILogger logger) { ArgumentNullException.ThrowIfNull(openCodeCliRunner); + ArgumentNullException.ThrowIfNull(playwrightCliInstaller); ArgumentNullException.ThrowIfNull(logger); _openCodeCliRunner = openCodeCliRunner; + _playwrightCliInstaller = playwrightCliInstaller; _logger = logger; } @@ -62,18 +67,8 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok _logger.LogDebug("Aspire MCP server is already configured"); } - // Add Playwright configuration callback if not already configured - if (!HasPlaywrightServerConfigured(configFilePath)) - { - _logger.LogDebug("Registering Playwright MCP configuration callback for OpenCode"); - CommonAgentApplicators.AddPlaywrightConfigurationCallback( - context, - ct => ApplyPlaywrightMcpConfigurationAsync(configDirectory, ct)); - } - else - { - _logger.LogDebug("Playwright MCP server is already configured"); - } + // Register Playwright CLI installation applicator + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); // Try to add skill file applicator for OpenCode CommonAgentApplicators.TryAddSkillFileApplicator( @@ -95,10 +90,8 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok _logger.LogDebug("Adding OpenCode applicator to create new opencode.jsonc at: {ConfigDirectory}", configDirectory.FullName); context.AddApplicator(CreateApplicator(configDirectory)); - // Register Playwright configuration callback - CommonAgentApplicators.AddPlaywrightConfigurationCallback( - context, - ct => ApplyPlaywrightMcpConfigurationAsync(configDirectory, ct)); + // Register Playwright CLI installation applicator + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); // Try to add skill file applicator for OpenCode CommonAgentApplicators.TryAddSkillFileApplicator( @@ -121,7 +114,32 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok /// True if the aspire server is already configured, false otherwise. private static bool HasAspireServerConfigured(string configFilePath) { - return McpConfigFileHelper.HasServerConfigured(configFilePath, "mcp", AspireServerName, RemoveJsonComments); + try + { + var content = File.ReadAllText(configFilePath); + + // Remove single-line comments for parsing (JSONC support) + content = RemoveJsonComments(content); + + var config = JsonNode.Parse(content)?.AsObject(); + + if (config is null) + { + return false; + } + + if (config.TryGetPropertyValue("mcp", out var mcpNode) && mcpNode is JsonObject mcp) + { + return mcp.ContainsKey(AspireServerName); + } + + return false; + } + catch (JsonException) + { + // If the JSON is malformed, assume aspire is not configured + return false; + } } /// @@ -180,8 +198,11 @@ private static async Task ApplyMcpConfigurationAsync( var configFilePath = Path.Combine(configDirectory.FullName, OpenCodeConfigFileName); var config = await McpConfigFileHelper.ReadConfigAsync(configFilePath, cancellationToken, RemoveJsonComments); - // Ensure schema is set for new files - config.TryAdd("$schema", "https://opencode.ai/config.json"); + // Ensure schema is set for new configs + if (!config.ContainsKey("$schema")) + { + config["$schema"] = "https://opencode.ai/config.json"; + } // Ensure "mcp" object exists if (!config.ContainsKey("mcp") || config["mcp"] is not JsonObject) @@ -204,45 +225,4 @@ private static async Task ApplyMcpConfigurationAsync( await File.WriteAllTextAsync(configFilePath, jsonOutput, cancellationToken); } - /// - /// Creates or updates the opencode.jsonc file with Playwright MCP configuration. - /// - private static async Task ApplyPlaywrightMcpConfigurationAsync( - DirectoryInfo configDirectory, - CancellationToken cancellationToken) - { - var configFilePath = Path.Combine(configDirectory.FullName, OpenCodeConfigFileName); - var config = await McpConfigFileHelper.ReadConfigAsync(configFilePath, cancellationToken, RemoveJsonComments); - - // Ensure schema is set for new files - config.TryAdd("$schema", "https://opencode.ai/config.json"); - - // Ensure "mcp" object exists - if (!config.ContainsKey("mcp") || config["mcp"] is not JsonObject) - { - config["mcp"] = new JsonObject(); - } - - var mcp = config["mcp"]!.AsObject(); - - // Add Playwright MCP server configuration - mcp["playwright"] = new JsonObject - { - ["type"] = "local", - ["command"] = new JsonArray("npx", "-y", "@playwright/mcp@latest"), - ["enabled"] = true - }; - - // Write the updated config using AOT-compatible serialization - var jsonOutput = JsonSerializer.Serialize(config, JsonSourceGenerationContext.Default.JsonObject); - await File.WriteAllTextAsync(configFilePath, jsonOutput, cancellationToken); - } - - /// - /// Checks if the Playwright MCP server is already configured in the opencode.jsonc file. - /// - private static bool HasPlaywrightServerConfigured(string configFilePath) - { - return McpConfigFileHelper.HasServerConfigured(configFilePath, "mcp", "playwright", RemoveJsonComments); - } } diff --git a/src/Aspire.Cli/Agents/Playwright/IPlaywrightCliRunner.cs b/src/Aspire.Cli/Agents/Playwright/IPlaywrightCliRunner.cs new file mode 100644 index 00000000000..047c73f6f85 --- /dev/null +++ b/src/Aspire.Cli/Agents/Playwright/IPlaywrightCliRunner.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Semver; + +namespace Aspire.Cli.Agents.Playwright; + +/// +/// Interface for running playwright-cli commands. +/// +internal interface IPlaywrightCliRunner +{ + /// + /// Gets the version of the playwright-cli if it is installed. + /// + /// A token to cancel the operation. + /// The version of the playwright-cli, or null if it is not installed. + Task GetVersionAsync(CancellationToken cancellationToken); + + /// + /// Installs Playwright CLI skill files into the workspace. + /// + /// A token to cancel the operation. + /// True if skill installation succeeded, false otherwise. + Task InstallSkillsAsync(CancellationToken cancellationToken); +} diff --git a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs new file mode 100644 index 00000000000..bad336ad486 --- /dev/null +++ b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs @@ -0,0 +1,154 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Security.Cryptography; +using Aspire.Cli.Npm; +using Microsoft.Extensions.Logging; +using Semver; + +namespace Aspire.Cli.Agents.Playwright; + +/// +/// Orchestrates secure installation of the Playwright CLI with supply chain verification. +/// +internal sealed class PlaywrightCliInstaller( + INpmRunner npmRunner, + IPlaywrightCliRunner playwrightCliRunner, + ILogger logger) +{ + /// + /// The npm package name for the Playwright CLI. + /// + internal const string PackageName = "@playwright/cli"; + + /// + /// The version range to resolve. Updated periodically with Aspire releases. + /// + internal const string VersionRange = "0.1"; + + /// + /// Installs the Playwright CLI with supply chain verification and generates skill files. + /// + /// A token to cancel the operation. + /// True if installation succeeded or was skipped (already up-to-date), false on failure. + public async Task InstallAsync(CancellationToken cancellationToken) + { + // Step 1: Resolve the target version and integrity hash from the npm registry. + logger.LogDebug("Resolving {Package}@{Range} from npm registry", PackageName, VersionRange); + var packageInfo = await npmRunner.ResolvePackageAsync(PackageName, VersionRange, cancellationToken); + + if (packageInfo is null) + { + logger.LogWarning("Failed to resolve {Package}@{Range} from npm registry. Is npm installed?", PackageName, VersionRange); + return false; + } + + logger.LogDebug("Resolved {Package}@{Version} with integrity {Integrity}", PackageName, packageInfo.Version, packageInfo.Integrity); + + // Step 2: Check if a suitable version is already installed. + var installedVersion = await playwrightCliRunner.GetVersionAsync(cancellationToken); + if (installedVersion is not null) + { + var comparison = SemVersion.ComparePrecedence(installedVersion, packageInfo.Version); + if (comparison >= 0) + { + logger.LogDebug( + "playwright-cli {InstalledVersion} is already installed (target: {TargetVersion}), skipping installation", + installedVersion, + packageInfo.Version); + + // Still install skills in case they're missing. + return await playwrightCliRunner.InstallSkillsAsync(cancellationToken); + } + + logger.LogDebug( + "Upgrading playwright-cli from {InstalledVersion} to {TargetVersion}", + installedVersion, + packageInfo.Version); + } + + // Step 3: Download the tarball via npm pack. + var tempDir = Path.Combine(Path.GetTempPath(), $"aspire-playwright-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + + try + { + logger.LogDebug("Downloading {Package}@{Version} to {TempDir}", PackageName, packageInfo.Version, tempDir); + var tarballPath = await npmRunner.PackAsync(PackageName, packageInfo.Version.ToString(), tempDir, cancellationToken); + + if (tarballPath is null) + { + logger.LogWarning("Failed to download {Package}@{Version}", PackageName, packageInfo.Version); + return false; + } + + // Step 4: Verify the downloaded tarball's SHA-512 hash matches the SRI integrity value. + if (!VerifyIntegrity(tarballPath, packageInfo.Integrity)) + { + logger.LogWarning( + "Integrity verification failed for {Package}@{Version}. The downloaded package may have been tampered with.", + PackageName, + packageInfo.Version); + return false; + } + + logger.LogDebug("Integrity verification passed for {TarballPath}", tarballPath); + + // Step 5: Run npm audit signatures for additional provenance verification. + var auditPassed = await npmRunner.AuditSignaturesAsync(cancellationToken); + if (!auditPassed) + { + logger.LogDebug("npm audit signatures did not pass, continuing with installation"); + } + + // Step 6: Install globally from the verified tarball. + logger.LogDebug("Installing {Package}@{Version} globally", PackageName, packageInfo.Version); + var installSuccess = await npmRunner.InstallGlobalAsync(tarballPath, cancellationToken); + + if (!installSuccess) + { + logger.LogWarning("Failed to install {Package}@{Version} globally", PackageName, packageInfo.Version); + return false; + } + + // Step 7: Generate skill files. + logger.LogDebug("Generating Playwright CLI skill files"); + return await playwrightCliRunner.InstallSkillsAsync(cancellationToken); + } + finally + { + // Clean up temporary directory. + try + { + if (Directory.Exists(tempDir)) + { + Directory.Delete(tempDir, recursive: true); + } + } + catch (IOException ex) + { + logger.LogDebug(ex, "Failed to clean up temporary directory: {TempDir}", tempDir); + } + } + } + + /// + /// Verifies that the SHA-512 hash of the file matches the SRI integrity string. + /// + internal static bool VerifyIntegrity(string filePath, string sriIntegrity) + { + // SRI format: "sha512-" + if (!sriIntegrity.StartsWith("sha512-", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var expectedHash = sriIntegrity["sha512-".Length..]; + + using var stream = File.OpenRead(filePath); + var hashBytes = SHA512.HashData(stream); + var actualHash = Convert.ToBase64String(hashBytes); + + return string.Equals(expectedHash, actualHash, StringComparison.Ordinal); + } +} diff --git a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliRunner.cs b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliRunner.cs new file mode 100644 index 00000000000..13fd127c4bb --- /dev/null +++ b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliRunner.cs @@ -0,0 +1,128 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using Semver; + +namespace Aspire.Cli.Agents.Playwright; + +/// +/// Runs playwright-cli commands. +/// +internal sealed class PlaywrightCliRunner(ILogger logger) : IPlaywrightCliRunner +{ + /// + public async Task GetVersionAsync(CancellationToken cancellationToken) + { + var executablePath = PathLookupHelper.FindFullPathFromPath("playwright-cli"); + if (executablePath is null) + { + logger.LogDebug("playwright-cli is not installed or not found in PATH"); + return null; + } + + try + { + var startInfo = new ProcessStartInfo(executablePath, "--version") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var process = new Process { StartInfo = startInfo }; + process.Start(); + + var outputTask = process.StandardOutput.ReadToEndAsync(cancellationToken); + var errorTask = process.StandardError.ReadToEndAsync(cancellationToken); + + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + var errorOutput = await errorTask.ConfigureAwait(false); + logger.LogDebug("playwright-cli --version returned non-zero exit code {ExitCode}: {Error}", process.ExitCode, errorOutput.Trim()); + return null; + } + + var output = await outputTask.ConfigureAwait(false); + var versionString = output.Trim().Split(['\n', '\r'], StringSplitOptions.RemoveEmptyEntries).FirstOrDefault()?.Trim(); + + if (string.IsNullOrEmpty(versionString)) + { + logger.LogDebug("playwright-cli returned empty version output"); + return null; + } + + if (versionString.StartsWith('v') || versionString.StartsWith('V')) + { + versionString = versionString[1..]; + } + + if (SemVersion.TryParse(versionString, SemVersionStyles.Any, out var version)) + { + logger.LogDebug("Found playwright-cli version: {Version}", version); + return version; + } + + logger.LogDebug("Could not parse playwright-cli version from output: {Output}", versionString); + return null; + } + catch (Exception ex) when (ex is InvalidOperationException or System.ComponentModel.Win32Exception) + { + logger.LogDebug(ex, "playwright-cli is not installed or not found in PATH"); + return null; + } + } + + /// + public async Task InstallSkillsAsync(CancellationToken cancellationToken) + { + var executablePath = PathLookupHelper.FindFullPathFromPath("playwright-cli"); + if (executablePath is null) + { + logger.LogDebug("playwright-cli is not installed or not found in PATH"); + return false; + } + + try + { + var startInfo = new ProcessStartInfo(executablePath) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + startInfo.ArgumentList.Add("install"); + startInfo.ArgumentList.Add("--skills"); + + using var process = new Process { StartInfo = startInfo }; + process.Start(); + + var outputTask = process.StandardOutput.ReadToEndAsync(cancellationToken); + var errorTask = process.StandardError.ReadToEndAsync(cancellationToken); + + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + var errorOutput = await errorTask.ConfigureAwait(false); + logger.LogDebug("playwright-cli install --skills returned non-zero exit code {ExitCode}: {Error}", process.ExitCode, errorOutput.Trim()); + return false; + } + + var output = await outputTask.ConfigureAwait(false); + logger.LogDebug("playwright-cli install --skills output: {Output}", output.Trim()); + return true; + } + catch (Exception ex) when (ex is InvalidOperationException or System.ComponentModel.Win32Exception) + { + logger.LogDebug(ex, "Failed to run playwright-cli install --skills"); + return false; + } + } +} diff --git a/src/Aspire.Cli/Agents/VsCode/VsCodeAgentEnvironmentScanner.cs b/src/Aspire.Cli/Agents/VsCode/VsCodeAgentEnvironmentScanner.cs index 22bbd1885e4..22c8774127d 100644 --- a/src/Aspire.Cli/Agents/VsCode/VsCodeAgentEnvironmentScanner.cs +++ b/src/Aspire.Cli/Agents/VsCode/VsCodeAgentEnvironmentScanner.cs @@ -3,6 +3,7 @@ using System.Text.Json; using System.Text.Json.Nodes; +using Aspire.Cli.Agents.Playwright; using Aspire.Cli.Resources; using Microsoft.Extensions.Logging; @@ -20,6 +21,7 @@ internal sealed class VsCodeAgentEnvironmentScanner : IAgentEnvironmentScanner private const string SkillFileDescription = "Create Aspire skill file (.github/skills/aspire/SKILL.md)"; private readonly IVsCodeCliRunner _vsCodeCliRunner; + private readonly PlaywrightCliInstaller _playwrightCliInstaller; private readonly CliExecutionContext _executionContext; private readonly ILogger _logger; @@ -27,14 +29,17 @@ internal sealed class VsCodeAgentEnvironmentScanner : IAgentEnvironmentScanner /// Initializes a new instance of . /// /// The VS Code CLI runner for checking if VS Code is installed. + /// The Playwright CLI installer for secure installation. /// The CLI execution context for accessing environment variables and settings. /// The logger for diagnostic output. - public VsCodeAgentEnvironmentScanner(IVsCodeCliRunner vsCodeCliRunner, CliExecutionContext executionContext, ILogger logger) + public VsCodeAgentEnvironmentScanner(IVsCodeCliRunner vsCodeCliRunner, PlaywrightCliInstaller playwrightCliInstaller, CliExecutionContext executionContext, ILogger logger) { ArgumentNullException.ThrowIfNull(vsCodeCliRunner); + ArgumentNullException.ThrowIfNull(playwrightCliInstaller); ArgumentNullException.ThrowIfNull(executionContext); ArgumentNullException.ThrowIfNull(logger); _vsCodeCliRunner = vsCodeCliRunner; + _playwrightCliInstaller = playwrightCliInstaller; _executionContext = executionContext; _logger = logger; } @@ -64,18 +69,8 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok _logger.LogDebug("Aspire MCP server is already configured in .vscode/mcp.json"); } - // Register Playwright configuration callback if not already configured - if (!HasPlaywrightServerConfigured(vsCodeFolder)) - { - _logger.LogDebug("Registering Playwright MCP configuration callback for .vscode folder"); - CommonAgentApplicators.AddPlaywrightConfigurationCallback( - context, - ct => ApplyPlaywrightMcpConfigurationAsync(vsCodeFolder, ct)); - } - else - { - _logger.LogDebug("Playwright MCP server is already configured in .vscode/mcp.json"); - } + // Register Playwright CLI installation applicator + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); // Try to add skill file applicator for GitHub Copilot CommonAgentApplicators.TryAddSkillFileApplicator( @@ -93,10 +88,8 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok _logger.LogDebug("Adding VS Code applicator for new .vscode folder at: {VsCodeFolder}", targetVsCodeFolder.FullName); context.AddApplicator(CreateAspireApplicator(targetVsCodeFolder)); - // Register Playwright configuration callback - CommonAgentApplicators.AddPlaywrightConfigurationCallback( - context, - ct => ApplyPlaywrightMcpConfigurationAsync(targetVsCodeFolder, ct)); + // Register Playwright CLI installation applicator + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); // Try to add skill file applicator for GitHub Copilot CommonAgentApplicators.TryAddSkillFileApplicator( @@ -205,16 +198,34 @@ private bool HasVsCodeEnvironmentVariables() private static bool HasAspireServerConfigured(DirectoryInfo vsCodeFolder) { var mcpConfigPath = Path.Combine(vsCodeFolder.FullName, McpConfigFileName); - return McpConfigFileHelper.HasServerConfigured(mcpConfigPath, "servers", AspireServerName); - } - /// - /// Checks if the Playwright MCP server is already configured in the mcp.json file. - /// - private static bool HasPlaywrightServerConfigured(DirectoryInfo vsCodeFolder) - { - var mcpConfigPath = Path.Combine(vsCodeFolder.FullName, McpConfigFileName); - return McpConfigFileHelper.HasServerConfigured(mcpConfigPath, "servers", "playwright"); + if (!File.Exists(mcpConfigPath)) + { + return false; + } + + try + { + var content = File.ReadAllText(mcpConfigPath); + var config = JsonNode.Parse(content)?.AsObject(); + + if (config is null) + { + return false; + } + + if (config.TryGetPropertyValue("servers", out var serversNode) && serversNode is JsonObject servers) + { + return servers.ContainsKey(AspireServerName); + } + + return false; + } + catch (JsonException) + { + // If the JSON is malformed, assume aspire is not configured + return false; + } } /// @@ -264,40 +275,4 @@ private static async Task ApplyAspireMcpConfigurationAsync( await File.WriteAllTextAsync(mcpConfigPath, jsonContent, cancellationToken); } - /// - /// Creates or updates the mcp.json file in the .vscode folder with Playwright MCP configuration. - /// - private static async Task ApplyPlaywrightMcpConfigurationAsync( - DirectoryInfo vsCodeFolder, - CancellationToken cancellationToken) - { - // Ensure the .vscode folder exists - if (!vsCodeFolder.Exists) - { - vsCodeFolder.Create(); - } - - var mcpConfigPath = Path.Combine(vsCodeFolder.FullName, McpConfigFileName); - var config = await McpConfigFileHelper.ReadConfigAsync(mcpConfigPath, cancellationToken); - - // Ensure "servers" object exists - if (!config.ContainsKey("servers") || config["servers"] is not JsonObject) - { - config["servers"] = new JsonObject(); - } - - var servers = config["servers"]!.AsObject(); - - // Add Playwright MCP server configuration - servers["playwright"] = new JsonObject - { - ["type"] = "stdio", - ["command"] = "npx", - ["args"] = new JsonArray("-y", "@playwright/mcp@latest") - }; - - // Write the updated config with indentation using AOT-compatible serialization - var jsonContent = JsonSerializer.Serialize(config, JsonSourceGenerationContext.Default.JsonObject); - await File.WriteAllTextAsync(mcpConfigPath, jsonContent, cancellationToken); - } } diff --git a/src/Aspire.Cli/Npm/INpmRunner.cs b/src/Aspire.Cli/Npm/INpmRunner.cs new file mode 100644 index 00000000000..582f40ffa70 --- /dev/null +++ b/src/Aspire.Cli/Npm/INpmRunner.cs @@ -0,0 +1,62 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Semver; + +namespace Aspire.Cli.Npm; + +/// +/// Represents the result of resolving an npm package version. +/// +internal sealed class NpmPackageInfo +{ + /// + /// Gets the resolved version of the package. + /// + public required SemVersion Version { get; init; } + + /// + /// Gets the SRI integrity hash (e.g., "sha512-...") for the package tarball. + /// + public required string Integrity { get; init; } +} + +/// +/// Interface for running npm CLI commands. +/// +internal interface INpmRunner +{ + /// + /// Resolves a package version and integrity hash from the npm registry. + /// + /// The npm package name (e.g., "@playwright/cli"). + /// The version range to resolve (e.g., "0.1"). + /// A token to cancel the operation. + /// The resolved package info, or null if the package was not found or npm is not installed. + Task ResolvePackageAsync(string packageName, string versionRange, CancellationToken cancellationToken); + + /// + /// Downloads a package tarball to a temporary directory using npm pack. + /// + /// The npm package name (e.g., "@playwright/cli"). + /// The exact version to download. + /// The directory to download the tarball into. + /// A token to cancel the operation. + /// The full path to the downloaded .tgz file, or null if the operation failed. + Task PackAsync(string packageName, string version, string outputDirectory, CancellationToken cancellationToken); + + /// + /// Runs npm audit signatures to verify package provenance. + /// + /// A token to cancel the operation. + /// True if the audit passed, false otherwise. + Task AuditSignaturesAsync(CancellationToken cancellationToken); + + /// + /// Installs a package globally from a local tarball file. + /// + /// The path to the .tgz file to install. + /// A token to cancel the operation. + /// True if the installation succeeded, false otherwise. + Task InstallGlobalAsync(string tarballPath, CancellationToken cancellationToken); +} diff --git a/src/Aspire.Cli/Npm/NpmRunner.cs b/src/Aspire.Cli/Npm/NpmRunner.cs new file mode 100644 index 00000000000..65b9ef0fa19 --- /dev/null +++ b/src/Aspire.Cli/Npm/NpmRunner.cs @@ -0,0 +1,186 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using Semver; + +namespace Aspire.Cli.Npm; + +/// +/// Runs npm CLI commands for package management operations. +/// +internal sealed class NpmRunner(ILogger logger) : INpmRunner +{ + /// + public async Task ResolvePackageAsync(string packageName, string versionRange, CancellationToken cancellationToken) + { + var npmPath = FindNpmPath(); + if (npmPath is null) + { + return null; + } + + // Resolve version: npm view @ version + var versionOutput = await RunNpmCommandAsync( + npmPath, + ["view", $"{packageName}@{versionRange}", "version"], + cancellationToken); + + if (versionOutput is null) + { + return null; + } + + var versionString = versionOutput.Trim(); + if (!SemVersion.TryParse(versionString, SemVersionStyles.Any, out var version)) + { + logger.LogDebug("Could not parse npm version from output: {Output}", versionString); + return null; + } + + // Resolve integrity hash: npm view @ dist.integrity + var integrityOutput = await RunNpmCommandAsync( + npmPath, + ["view", $"{packageName}@{version}", "dist.integrity"], + cancellationToken); + + if (string.IsNullOrWhiteSpace(integrityOutput)) + { + logger.LogDebug("Could not resolve integrity hash for {Package}@{Version}", packageName, version); + return null; + } + + return new NpmPackageInfo + { + Version = version, + Integrity = integrityOutput.Trim() + }; + } + + /// + public async Task PackAsync(string packageName, string version, string outputDirectory, CancellationToken cancellationToken) + { + var npmPath = FindNpmPath(); + if (npmPath is null) + { + return null; + } + + var output = await RunNpmCommandAsync( + npmPath, + ["pack", $"{packageName}@{version}", "--pack-destination", outputDirectory], + cancellationToken); + + if (output is null) + { + return null; + } + + // npm pack outputs the filename of the created tarball + var filename = output.Trim().Split(['\n', '\r'], StringSplitOptions.RemoveEmptyEntries).LastOrDefault(); + if (string.IsNullOrWhiteSpace(filename)) + { + logger.LogDebug("npm pack returned empty filename"); + return null; + } + + var tarballPath = Path.Combine(outputDirectory, filename); + if (!File.Exists(tarballPath)) + { + logger.LogDebug("npm pack output file not found: {Path}", tarballPath); + return null; + } + + return tarballPath; + } + + /// + public async Task AuditSignaturesAsync(CancellationToken cancellationToken) + { + var npmPath = FindNpmPath(); + if (npmPath is null) + { + return false; + } + + var output = await RunNpmCommandAsync( + npmPath, + ["audit", "signatures"], + cancellationToken); + + return output is not null; + } + + /// + public async Task InstallGlobalAsync(string tarballPath, CancellationToken cancellationToken) + { + var npmPath = FindNpmPath(); + if (npmPath is null) + { + return false; + } + + var output = await RunNpmCommandAsync( + npmPath, + ["install", "-g", tarballPath], + cancellationToken); + + return output is not null; + } + + private string? FindNpmPath() + { + var npmPath = PathLookupHelper.FindFullPathFromPath("npm"); + if (npmPath is null) + { + logger.LogDebug("npm is not installed or not found in PATH"); + } + + return npmPath; + } + + private async Task RunNpmCommandAsync(string npmPath, string[] args, CancellationToken cancellationToken) + { + var argsString = string.Join(" ", args); + logger.LogDebug("Running npm {Args}", argsString); + + try + { + var startInfo = new ProcessStartInfo(npmPath) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + foreach (var arg in args) + { + startInfo.ArgumentList.Add(arg); + } + + using var process = new Process { StartInfo = startInfo }; + process.Start(); + + var outputTask = process.StandardOutput.ReadToEndAsync(cancellationToken); + var errorTask = process.StandardError.ReadToEndAsync(cancellationToken); + + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + var errorOutput = await errorTask.ConfigureAwait(false); + logger.LogDebug("npm {Args} returned non-zero exit code {ExitCode}: {Error}", argsString, process.ExitCode, errorOutput.Trim()); + return null; + } + + return await outputTask.ConfigureAwait(false); + } + catch (Exception ex) when (ex is InvalidOperationException or System.ComponentModel.Win32Exception) + { + logger.LogDebug(ex, "Failed to run npm {Args}", argsString); + return null; + } + } +} diff --git a/src/Aspire.Cli/Program.cs b/src/Aspire.Cli/Program.cs index ebc0c3ea7c1..518b072beb1 100644 --- a/src/Aspire.Cli/Program.cs +++ b/src/Aspire.Cli/Program.cs @@ -326,6 +326,11 @@ internal static async Task BuildApplicationAsync(string[] args, Dictionar builder.Services.AddSingleton(); builder.Services.AddSingleton(); + // Npm and Playwright CLI operations. + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + // Agent environment detection. builder.Services.AddSingleton(); builder.Services.TryAddEnumerable(ServiceDescriptor.Singleton()); diff --git a/src/Aspire.Cli/Resources/McpCommandStrings.Designer.cs b/src/Aspire.Cli/Resources/McpCommandStrings.Designer.cs index e347734caf0..69cb7ac3cfe 100644 --- a/src/Aspire.Cli/Resources/McpCommandStrings.Designer.cs +++ b/src/Aspire.Cli/Resources/McpCommandStrings.Designer.cs @@ -106,7 +106,7 @@ internal static string InitCommand_ConfigurationComplete { } /// - /// Looks up a localized string similar to Pre-configure Playwright MCP server?. + /// Looks up a localized string similar to Install Playwright CLI for browser automation?. /// internal static string InitCommand_ConfigurePlaywrightPrompt { get { diff --git a/src/Aspire.Cli/Resources/McpCommandStrings.resx b/src/Aspire.Cli/Resources/McpCommandStrings.resx index 3ecae62c8a9..e5624ec847e 100644 --- a/src/Aspire.Cli/Resources/McpCommandStrings.resx +++ b/src/Aspire.Cli/Resources/McpCommandStrings.resx @@ -100,6 +100,6 @@ Create agent environment specific instruction files? - Pre-configure Playwright MCP server? + Install Playwright CLI for browser automation? diff --git a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.cs.xlf b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.cs.xlf index 8971878c481..3dbd0abdd6c 100644 --- a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.cs.xlf +++ b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.cs.xlf @@ -28,8 +28,8 @@ - Pre-configure Playwright MCP server? - Chcete předem nakonfigurovat server Playwright MCP? + Install Playwright CLI for browser automation? + Chcete předem nakonfigurovat server Playwright MCP? diff --git a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.de.xlf b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.de.xlf index 520d43c5744..56a88563b4d 100644 --- a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.de.xlf +++ b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.de.xlf @@ -28,8 +28,8 @@ - Pre-configure Playwright MCP server? - Playwright MCP-Server vorkonfigurieren? + Install Playwright CLI for browser automation? + Playwright MCP-Server vorkonfigurieren? diff --git a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.es.xlf b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.es.xlf index ce9e8371562..ca14d274ada 100644 --- a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.es.xlf +++ b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.es.xlf @@ -28,8 +28,8 @@ - Pre-configure Playwright MCP server? - ¿Configurar previamente el servidor Playwright MCP? + Install Playwright CLI for browser automation? + ¿Configurar previamente el servidor Playwright MCP? diff --git a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.fr.xlf b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.fr.xlf index 3538476e647..b230ee0b11c 100644 --- a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.fr.xlf +++ b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.fr.xlf @@ -28,8 +28,8 @@ - Pre-configure Playwright MCP server? - Préconfigurer le serveur de MCP Playwright ? + Install Playwright CLI for browser automation? + Préconfigurer le serveur de MCP Playwright ? diff --git a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.it.xlf b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.it.xlf index 4d0349bcd93..6e68d5f22c0 100644 --- a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.it.xlf +++ b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.it.xlf @@ -28,8 +28,8 @@ - Pre-configure Playwright MCP server? - Preconfigurare il server MCP di Playwright? + Install Playwright CLI for browser automation? + Preconfigurare il server MCP di Playwright? diff --git a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.ja.xlf b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.ja.xlf index fc8afa58fbe..1eb7bd95b3f 100644 --- a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.ja.xlf +++ b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.ja.xlf @@ -28,8 +28,8 @@ - Pre-configure Playwright MCP server? - Playwright MCP サーバーを事前に構成しますか? + Install Playwright CLI for browser automation? + Playwright MCP サーバーを事前に構成しますか? diff --git a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.ko.xlf b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.ko.xlf index ff56c690c0c..1b2e7bbd80c 100644 --- a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.ko.xlf +++ b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.ko.xlf @@ -28,8 +28,8 @@ - Pre-configure Playwright MCP server? - Playwright MCP 서버를 미리 구성할까요? + Install Playwright CLI for browser automation? + Playwright MCP 서버를 미리 구성할까요? diff --git a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.pl.xlf b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.pl.xlf index 44e5c0d32b0..e6067823e2e 100644 --- a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.pl.xlf +++ b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.pl.xlf @@ -28,8 +28,8 @@ - Pre-configure Playwright MCP server? - Wstępnie skonfigurować serwer MCP Playwright? + Install Playwright CLI for browser automation? + Wstępnie skonfigurować serwer MCP Playwright? diff --git a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.pt-BR.xlf b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.pt-BR.xlf index b6a7a8f1bca..ffe5cdd3c25 100644 --- a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.pt-BR.xlf +++ b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.pt-BR.xlf @@ -28,8 +28,8 @@ - Pre-configure Playwright MCP server? - Pré-configurar o servidor MCP do Playwright? + Install Playwright CLI for browser automation? + Pré-configurar o servidor MCP do Playwright? diff --git a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.ru.xlf b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.ru.xlf index 961b13be6e8..cb399ce4199 100644 --- a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.ru.xlf +++ b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.ru.xlf @@ -28,8 +28,8 @@ - Pre-configure Playwright MCP server? - Предварительно настроить сервер MCP Playwright? + Install Playwright CLI for browser automation? + Предварительно настроить сервер MCP Playwright? diff --git a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.tr.xlf b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.tr.xlf index c5fe900d32b..e1d76049542 100644 --- a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.tr.xlf +++ b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.tr.xlf @@ -28,8 +28,8 @@ - Pre-configure Playwright MCP server? - Playwright MCP sunucusunda ön yapılandırma yapılsın mı? + Install Playwright CLI for browser automation? + Playwright MCP sunucusunda ön yapılandırma yapılsın mı? diff --git a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.zh-Hans.xlf b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.zh-Hans.xlf index b037114d647..0faafc9773f 100644 --- a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.zh-Hans.xlf +++ b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.zh-Hans.xlf @@ -28,8 +28,8 @@ - Pre-configure Playwright MCP server? - 预配置 Playwright MCP 服务器? + Install Playwright CLI for browser automation? + 预配置 Playwright MCP 服务器? diff --git a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.zh-Hant.xlf b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.zh-Hant.xlf index 77d090531fe..ef26fe6d098 100644 --- a/src/Aspire.Cli/Resources/xlf/McpCommandStrings.zh-Hant.xlf +++ b/src/Aspire.Cli/Resources/xlf/McpCommandStrings.zh-Hant.xlf @@ -28,8 +28,8 @@ - Pre-configure Playwright MCP server? - 預先設定 Playwright MCP 伺服器? + Install Playwright CLI for browser automation? + 預先設定 Playwright MCP 伺服器? diff --git a/tests/Aspire.Cli.Tests/Agents/ClaudeCodeAgentEnvironmentScannerTests.cs b/tests/Aspire.Cli.Tests/Agents/ClaudeCodeAgentEnvironmentScannerTests.cs index a0a0185e653..1866074d129 100644 --- a/tests/Aspire.Cli.Tests/Agents/ClaudeCodeAgentEnvironmentScannerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/ClaudeCodeAgentEnvironmentScannerTests.cs @@ -4,6 +4,8 @@ using Microsoft.AspNetCore.InternalTesting; using Aspire.Cli.Agents; using Aspire.Cli.Agents.ClaudeCode; +using Aspire.Cli.Agents.Playwright; +using Aspire.Cli.Tests.TestServices; using Aspire.Cli.Tests.Utils; using Microsoft.Extensions.Logging.Abstractions; using Semver; @@ -24,7 +26,7 @@ public async Task ApplyAsync_WithMalformedMcpJson_ThrowsInvalidOperationExceptio var claudeCodeCliRunner = new FakeClaudeCodeCliRunner(new SemVersion(1, 0, 0)); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new ClaudeCodeAgentEnvironmentScanner(claudeCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new ClaudeCodeAgentEnvironmentScanner(claudeCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -52,7 +54,7 @@ public async Task ApplyAsync_WithEmptyMcpJson_ThrowsInvalidOperationException() var claudeCodeCliRunner = new FakeClaudeCodeCliRunner(new SemVersion(1, 0, 0)); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new ClaudeCodeAgentEnvironmentScanner(claudeCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new ClaudeCodeAgentEnvironmentScanner(claudeCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -78,7 +80,7 @@ public async Task ApplyAsync_WithMalformedMcpJson_DoesNotOverwriteFile() var claudeCodeCliRunner = new FakeClaudeCodeCliRunner(new SemVersion(1, 0, 0)); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new ClaudeCodeAgentEnvironmentScanner(claudeCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new ClaudeCodeAgentEnvironmentScanner(claudeCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -118,6 +120,14 @@ private static CliExecutionContext CreateExecutionContext(DirectoryInfo workingD homeDirectory: workingDirectory); } + private static PlaywrightCliInstaller CreatePlaywrightCliInstaller() + { + return new PlaywrightCliInstaller( + new FakeNpmRunner(), + new FakePlaywrightCliRunner(), + NullLogger.Instance); + } + private sealed class FakeClaudeCodeCliRunner(SemVersion? version) : IClaudeCodeCliRunner { public Task GetVersionAsync(CancellationToken cancellationToken) diff --git a/tests/Aspire.Cli.Tests/Agents/CopilotCliAgentEnvironmentScannerTests.cs b/tests/Aspire.Cli.Tests/Agents/CopilotCliAgentEnvironmentScannerTests.cs index 3a791f21202..a0375a2c5e6 100644 --- a/tests/Aspire.Cli.Tests/Agents/CopilotCliAgentEnvironmentScannerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/CopilotCliAgentEnvironmentScannerTests.cs @@ -5,7 +5,9 @@ using System.Text.Json.Nodes; using Aspire.Cli.Agents; using Aspire.Cli.Agents.CopilotCli; +using Aspire.Cli.Agents.Playwright; using Aspire.Cli.Tests.Utils; +using Aspire.Cli.Tests.TestServices; using Microsoft.Extensions.Logging.Abstractions; using Semver; @@ -19,12 +21,12 @@ public async Task ScanAsync_WhenCopilotCliInstalled_ReturnsApplicator() using var workspace = TemporaryWorkspace.Create(outputHelper); var copilotCliRunner = new FakeCopilotCliRunner(new SemVersion(1, 0, 0)); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, executionContext, NullLogger.Instance); + var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); - // Scanner adds applicators for: Aspire MCP, Playwright MCP, and agent instructions + // Scanner adds applicators for: Aspire MCP, Playwright CLI, and agent instructions Assert.NotEmpty(context.Applicators); Assert.Contains(context.Applicators, a => a.Description.Contains("GitHub Copilot CLI")); } @@ -40,12 +42,12 @@ public async Task ApplyAsync_CreatesMcpConfigJsonWithCorrectConfiguration() // Create a scanner that writes to a known test location var copilotCliRunner = new FakeCopilotCliRunner(new SemVersion(1, 0, 0)); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, executionContext, NullLogger.Instance); + var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); - // Scanner adds applicators for: Aspire MCP, Playwright MCP, and agent instructions + // Scanner adds applicators for: Aspire MCP, Playwright CLI, and agent instructions Assert.NotEmpty(context.Applicators); var aspireApplicator = context.Applicators.First(a => a.Description.Contains("Aspire MCP")); @@ -109,7 +111,7 @@ public async Task ApplyAsync_PreservesExistingMcpConfigContent() var copilotCliRunner = new FakeCopilotCliRunner(new SemVersion(1, 0, 0)); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, executionContext, NullLogger.Instance); + var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -128,7 +130,7 @@ public async Task ApplyAsync_PreservesExistingMcpConfigContent() } [Fact] - public async Task ScanAsync_WhenAspireAlreadyConfigured_ReturnsNoApplicator() + public async Task ScanAsync_WhenAspireAlreadyConfigured_ReturnsPlaywrightCliApplicatorOnly() { using var workspace = TemporaryWorkspace.Create(outputHelper); var copilotFolder = workspace.CreateDirectory(".copilot"); @@ -141,10 +143,6 @@ public async Task ScanAsync_WhenAspireAlreadyConfigured_ReturnsNoApplicator() ["aspire"] = new JsonObject { ["command"] = "aspire" - }, - ["playwright"] = new JsonObject - { - ["command"] = "npx" } } }; @@ -158,13 +156,14 @@ public async Task ScanAsync_WhenAspireAlreadyConfigured_ReturnsNoApplicator() var copilotCliRunner = new FakeCopilotCliRunner(new SemVersion(1, 0, 0)); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, executionContext, NullLogger.Instance); + var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); - // No applicators should be returned since Aspire MCP, Playwright MCP are configured and skill file exists with same content - Assert.Empty(context.Applicators); + // Only the Playwright CLI applicator should be offered (Aspire MCP is configured, skill file is up to date) + Assert.Single(context.Applicators); + Assert.Contains(context.Applicators, a => a.Description.Contains("Playwright CLI")); } [Fact] @@ -173,12 +172,12 @@ public async Task ScanAsync_WhenInVSCode_ReturnsApplicatorWithoutCallingRunner() using var workspace = TemporaryWorkspace.Create(outputHelper); var copilotCliRunner = new FakeCopilotCliRunner(null); // Return null to verify it's not called var executionContext = CreateExecutionContextWithVSCode(workspace.WorkspaceRoot); - var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, executionContext, NullLogger.Instance); + var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); - // Scanner adds applicators for: Aspire MCP, Playwright MCP, and agent instructions + // Scanner adds applicators for: Aspire MCP, Playwright CLI, and agent instructions Assert.NotEmpty(context.Applicators); Assert.Contains(context.Applicators, a => a.Description.Contains("GitHub Copilot")); Assert.False(copilotCliRunner.WasCalled); // Verify GetVersionAsync was not called @@ -239,7 +238,7 @@ public async Task ApplyAsync_WithMalformedMcpJson_ThrowsInvalidOperationExceptio var copilotCliRunner = new FakeCopilotCliRunner(new SemVersion(1, 0, 0)); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, executionContext, NullLogger.Instance); + var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -267,7 +266,7 @@ public async Task ApplyAsync_WithEmptyMcpJson_ThrowsInvalidOperationException() var copilotCliRunner = new FakeCopilotCliRunner(new SemVersion(1, 0, 0)); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, executionContext, NullLogger.Instance); + var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -293,7 +292,7 @@ public async Task ApplyAsync_WithMalformedMcpJson_DoesNotOverwriteFile() var copilotCliRunner = new FakeCopilotCliRunner(new SemVersion(1, 0, 0)); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, executionContext, NullLogger.Instance); + var scanner = new CopilotCliAgentEnvironmentScanner(copilotCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -322,4 +321,12 @@ private sealed class FakeCopilotCliRunner(SemVersion? version) : ICopilotCliRunn return Task.FromResult(version); } } + + private static PlaywrightCliInstaller CreatePlaywrightCliInstaller() + { + return new PlaywrightCliInstaller( + new FakeNpmRunner(), + new FakePlaywrightCliRunner(), + NullLogger.Instance); + } } diff --git a/tests/Aspire.Cli.Tests/Agents/OpenCodeAgentEnvironmentScannerTests.cs b/tests/Aspire.Cli.Tests/Agents/OpenCodeAgentEnvironmentScannerTests.cs index 0e8b4540592..6e5c40542b1 100644 --- a/tests/Aspire.Cli.Tests/Agents/OpenCodeAgentEnvironmentScannerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/OpenCodeAgentEnvironmentScannerTests.cs @@ -4,6 +4,8 @@ using Microsoft.AspNetCore.InternalTesting; using Aspire.Cli.Agents; using Aspire.Cli.Agents.OpenCode; +using Aspire.Cli.Agents.Playwright; +using Aspire.Cli.Tests.TestServices; using Aspire.Cli.Tests.Utils; using Microsoft.Extensions.Logging.Abstractions; using Semver; @@ -22,7 +24,7 @@ public async Task ApplyAsync_WithMalformedOpenCodeJsonc_ThrowsInvalidOperationEx await File.WriteAllTextAsync(configPath, "{ invalid json content"); var openCodeCliRunner = new FakeOpenCodeCliRunner(new SemVersion(1, 0, 0)); - var scanner = new OpenCodeAgentEnvironmentScanner(openCodeCliRunner, NullLogger.Instance); + var scanner = new OpenCodeAgentEnvironmentScanner(openCodeCliRunner, CreatePlaywrightCliInstaller(), NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -48,7 +50,7 @@ public async Task ApplyAsync_WithEmptyOpenCodeJsonc_ThrowsInvalidOperationExcept await File.WriteAllTextAsync(configPath, ""); var openCodeCliRunner = new FakeOpenCodeCliRunner(new SemVersion(1, 0, 0)); - var scanner = new OpenCodeAgentEnvironmentScanner(openCodeCliRunner, NullLogger.Instance); + var scanner = new OpenCodeAgentEnvironmentScanner(openCodeCliRunner, CreatePlaywrightCliInstaller(), NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -72,7 +74,7 @@ public async Task ApplyAsync_WithMalformedOpenCodeJsonc_DoesNotOverwriteFile() await File.WriteAllTextAsync(configPath, originalContent); var openCodeCliRunner = new FakeOpenCodeCliRunner(new SemVersion(1, 0, 0)); - var scanner = new OpenCodeAgentEnvironmentScanner(openCodeCliRunner, NullLogger.Instance); + var scanner = new OpenCodeAgentEnvironmentScanner(openCodeCliRunner, CreatePlaywrightCliInstaller(), NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -98,6 +100,14 @@ private static AgentEnvironmentScanContext CreateScanContext( }; } + private static PlaywrightCliInstaller CreatePlaywrightCliInstaller() + { + return new PlaywrightCliInstaller( + new FakeNpmRunner(), + new FakePlaywrightCliRunner(), + NullLogger.Instance); + } + private sealed class FakeOpenCodeCliRunner(SemVersion? version) : IOpenCodeCliRunner { public Task GetVersionAsync(CancellationToken cancellationToken) diff --git a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs new file mode 100644 index 00000000000..a3756ef40bf --- /dev/null +++ b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs @@ -0,0 +1,338 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Security.Cryptography; +using Aspire.Cli.Agents.Playwright; +using Aspire.Cli.Npm; +using Microsoft.Extensions.Logging.Abstractions; +using Semver; + +namespace Aspire.Cli.Tests.Agents; + +public class PlaywrightCliInstallerTests +{ + [Fact] + public async Task InstallAsync_WhenNpmResolveReturnsNull_ReturnsFalse() + { + var npmRunner = new TestNpmRunner + { + ResolveResult = null + }; + var playwrightRunner = new TestPlaywrightCliRunner(); + var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + + var result = await installer.InstallAsync(CancellationToken.None); + + Assert.False(result); + } + + [Fact] + public async Task InstallAsync_WhenAlreadyInstalledAtSameVersion_SkipsInstallAndInstallsSkills() + { + var version = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); + var npmRunner = new TestNpmRunner + { + ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-abc123" } + }; + var playwrightRunner = new TestPlaywrightCliRunner + { + InstalledVersion = version, + InstallSkillsResult = true + }; + var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + + var result = await installer.InstallAsync(CancellationToken.None); + + Assert.True(result); + Assert.True(playwrightRunner.InstallSkillsCalled); + Assert.False(npmRunner.PackCalled); + Assert.False(npmRunner.InstallGlobalCalled); + } + + [Fact] + public async Task InstallAsync_WhenNewerVersionInstalled_SkipsInstallAndInstallsSkills() + { + var targetVersion = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); + var installedVersion = SemVersion.Parse("0.2.0", SemVersionStyles.Strict); + var npmRunner = new TestNpmRunner + { + ResolveResult = new NpmPackageInfo { Version = targetVersion, Integrity = "sha512-abc123" } + }; + var playwrightRunner = new TestPlaywrightCliRunner + { + InstalledVersion = installedVersion, + InstallSkillsResult = true + }; + var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + + var result = await installer.InstallAsync(CancellationToken.None); + + Assert.True(result); + Assert.True(playwrightRunner.InstallSkillsCalled); + Assert.False(npmRunner.PackCalled); + } + + [Fact] + public async Task InstallAsync_WhenPackFails_ReturnsFalse() + { + var version = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); + var npmRunner = new TestNpmRunner + { + ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-abc123" }, + PackResult = null + }; + var playwrightRunner = new TestPlaywrightCliRunner(); + var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + + var result = await installer.InstallAsync(CancellationToken.None); + + Assert.False(result); + Assert.True(npmRunner.PackCalled); + } + + [Fact] + public async Task InstallAsync_WhenIntegrityCheckFails_ReturnsFalse() + { + var version = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); + // Create a temp file with known content and a non-matching hash + var tempDir = Path.Combine(Path.GetTempPath(), $"test-playwright-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + var tarballPath = Path.Combine(tempDir, "package.tgz"); + await File.WriteAllBytesAsync(tarballPath, [1, 2, 3]); + + try + { + var npmRunner = new TestNpmRunner + { + ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-definitelyWrongHash" }, + PackResult = tarballPath + }; + var playwrightRunner = new TestPlaywrightCliRunner(); + var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + + var result = await installer.InstallAsync(CancellationToken.None); + + Assert.False(result); + Assert.False(npmRunner.InstallGlobalCalled); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public async Task InstallAsync_WhenIntegrityCheckPasses_InstallsGlobally() + { + var version = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); + var tempDir = Path.Combine(Path.GetTempPath(), $"test-playwright-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + var tarballPath = Path.Combine(tempDir, "package.tgz"); + var content = new byte[] { 10, 20, 30, 40, 50 }; + await File.WriteAllBytesAsync(tarballPath, content); + + // Compute the correct SRI hash for the content + var hash = SHA512.HashData(content); + var integrity = $"sha512-{Convert.ToBase64String(hash)}"; + + try + { + var npmRunner = new TestNpmRunner + { + ResolveResult = new NpmPackageInfo { Version = version, Integrity = integrity }, + PackResult = tarballPath, + InstallGlobalResult = true, + AuditResult = true + }; + var playwrightRunner = new TestPlaywrightCliRunner + { + InstallSkillsResult = true + }; + var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + + var result = await installer.InstallAsync(CancellationToken.None); + + Assert.True(result); + Assert.True(npmRunner.InstallGlobalCalled); + Assert.True(playwrightRunner.InstallSkillsCalled); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public async Task InstallAsync_WhenGlobalInstallFails_ReturnsFalse() + { + var version = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); + var tempDir = Path.Combine(Path.GetTempPath(), $"test-playwright-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + var tarballPath = Path.Combine(tempDir, "package.tgz"); + var content = new byte[] { 10, 20, 30 }; + await File.WriteAllBytesAsync(tarballPath, content); + + var hash = SHA512.HashData(content); + var integrity = $"sha512-{Convert.ToBase64String(hash)}"; + + try + { + var npmRunner = new TestNpmRunner + { + ResolveResult = new NpmPackageInfo { Version = version, Integrity = integrity }, + PackResult = tarballPath, + InstallGlobalResult = false + }; + var playwrightRunner = new TestPlaywrightCliRunner(); + var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + + var result = await installer.InstallAsync(CancellationToken.None); + + Assert.False(result); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public async Task InstallAsync_WhenOlderVersionInstalled_PerformsUpgrade() + { + var targetVersion = SemVersion.Parse("0.1.2", SemVersionStyles.Strict); + var installedVersion = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); + var tempDir = Path.Combine(Path.GetTempPath(), $"test-playwright-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + var tarballPath = Path.Combine(tempDir, "package.tgz"); + var content = new byte[] { 99, 100 }; + await File.WriteAllBytesAsync(tarballPath, content); + + var hash = SHA512.HashData(content); + var integrity = $"sha512-{Convert.ToBase64String(hash)}"; + + try + { + var npmRunner = new TestNpmRunner + { + ResolveResult = new NpmPackageInfo { Version = targetVersion, Integrity = integrity }, + PackResult = tarballPath, + InstallGlobalResult = true, + AuditResult = true + }; + var playwrightRunner = new TestPlaywrightCliRunner + { + InstalledVersion = installedVersion, + InstallSkillsResult = true + }; + var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + + var result = await installer.InstallAsync(CancellationToken.None); + + Assert.True(result); + Assert.True(npmRunner.PackCalled); + Assert.True(npmRunner.InstallGlobalCalled); + Assert.True(playwrightRunner.InstallSkillsCalled); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [Fact] + public void VerifyIntegrity_WithMatchingHash_ReturnsTrue() + { + var tempPath = Path.GetTempFileName(); + try + { + var content = "test content for hashing"u8.ToArray(); + File.WriteAllBytes(tempPath, content); + + var hash = SHA512.HashData(content); + var integrity = $"sha512-{Convert.ToBase64String(hash)}"; + + Assert.True(PlaywrightCliInstaller.VerifyIntegrity(tempPath, integrity)); + } + finally + { + File.Delete(tempPath); + } + } + + [Fact] + public void VerifyIntegrity_WithNonMatchingHash_ReturnsFalse() + { + var tempPath = Path.GetTempFileName(); + try + { + File.WriteAllText(tempPath, "some content"); + + Assert.False(PlaywrightCliInstaller.VerifyIntegrity(tempPath, "sha512-wronghash")); + } + finally + { + File.Delete(tempPath); + } + } + + [Fact] + public void VerifyIntegrity_WithNonSha512Prefix_ReturnsFalse() + { + var tempPath = Path.GetTempFileName(); + try + { + File.WriteAllText(tempPath, "some content"); + + Assert.False(PlaywrightCliInstaller.VerifyIntegrity(tempPath, "sha256-somehash")); + } + finally + { + File.Delete(tempPath); + } + } + + private sealed class TestNpmRunner : INpmRunner + { + public NpmPackageInfo? ResolveResult { get; set; } + public string? PackResult { get; set; } + public bool AuditResult { get; set; } = true; + public bool InstallGlobalResult { get; set; } = true; + + public bool PackCalled { get; private set; } + public bool InstallGlobalCalled { get; private set; } + + public Task ResolvePackageAsync(string packageName, string versionRange, CancellationToken cancellationToken) + => Task.FromResult(ResolveResult); + + public Task PackAsync(string packageName, string version, string outputDirectory, CancellationToken cancellationToken) + { + PackCalled = true; + return Task.FromResult(PackResult); + } + + public Task AuditSignaturesAsync(CancellationToken cancellationToken) + => Task.FromResult(AuditResult); + + public Task InstallGlobalAsync(string tarballPath, CancellationToken cancellationToken) + { + InstallGlobalCalled = true; + return Task.FromResult(InstallGlobalResult); + } + } + + private sealed class TestPlaywrightCliRunner : IPlaywrightCliRunner + { + public SemVersion? InstalledVersion { get; set; } + public bool InstallSkillsResult { get; set; } + public bool InstallSkillsCalled { get; private set; } + + public Task GetVersionAsync(CancellationToken cancellationToken) + => Task.FromResult(InstalledVersion); + + public Task InstallSkillsAsync(CancellationToken cancellationToken) + { + InstallSkillsCalled = true; + return Task.FromResult(InstallSkillsResult); + } + } +} diff --git a/tests/Aspire.Cli.Tests/Agents/VsCodeAgentEnvironmentScannerTests.cs b/tests/Aspire.Cli.Tests/Agents/VsCodeAgentEnvironmentScannerTests.cs index 989170312cd..92ab5d41b10 100644 --- a/tests/Aspire.Cli.Tests/Agents/VsCodeAgentEnvironmentScannerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/VsCodeAgentEnvironmentScannerTests.cs @@ -4,8 +4,10 @@ using Microsoft.AspNetCore.InternalTesting; using System.Text.Json.Nodes; using Aspire.Cli.Agents; +using Aspire.Cli.Agents.Playwright; using Aspire.Cli.Agents.VsCode; using Aspire.Cli.Tests.Utils; +using Aspire.Cli.Tests.TestServices; using Microsoft.Extensions.Logging.Abstractions; using Semver; @@ -20,12 +22,12 @@ public async Task ScanAsync_WhenVsCodeFolderExists_ReturnsApplicator() var vsCodeFolder = workspace.CreateDirectory(".vscode"); var vsCodeCliRunner = new FakeVsCodeCliRunner(null); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); - // Scanner adds applicators for: Aspire MCP, Playwright MCP, and agent instructions + // Scanner adds applicators for: Aspire MCP, Playwright CLI, and agent instructions Assert.NotEmpty(context.Applicators); Assert.Contains(context.Applicators, a => a.Description.Contains("VS Code")); } @@ -38,12 +40,12 @@ public async Task ScanAsync_WhenVsCodeFolderExistsInParent_ReturnsApplicatorForP var childDir = workspace.CreateDirectory("subdir"); var vsCodeCliRunner = new FakeVsCodeCliRunner(null); var executionContext = CreateExecutionContext(childDir); - var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(childDir, workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); - // Scanner adds applicators for: Aspire MCP, Playwright MCP, and agent instructions + // Scanner adds applicators for: Aspire MCP, Playwright CLI, and agent instructions Assert.NotEmpty(context.Applicators); Assert.Contains(context.Applicators, a => a.Description.Contains("VS Code")); } @@ -56,7 +58,7 @@ public async Task ScanAsync_WhenRepositoryRootReachedBeforeVsCode_AndNoCliAvaila // Repository root is the workspace root, so search should stop there var vsCodeCliRunner = new FakeVsCodeCliRunner(null); var executionContext = CreateExecutionContext(childDir); - var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(childDir, workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -70,12 +72,12 @@ public async Task ScanAsync_WhenNoVsCodeFolder_AndVsCodeCliAvailable_ReturnsAppl using var workspace = TemporaryWorkspace.Create(outputHelper); var vsCodeCliRunner = new FakeVsCodeCliRunner(new SemVersion(1, 85, 0)); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); - // Scanner adds applicators for: Aspire MCP, Playwright MCP, and agent instructions + // Scanner adds applicators for: Aspire MCP, Playwright CLI, and agent instructions Assert.NotEmpty(context.Applicators); Assert.Contains(context.Applicators, a => a.Description.Contains("VS Code")); } @@ -86,7 +88,7 @@ public async Task ScanAsync_WhenNoVsCodeFolder_AndNoCliAvailable_ReturnsNoApplic using var workspace = TemporaryWorkspace.Create(outputHelper); var vsCodeCliRunner = new FakeVsCodeCliRunner(null); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); // This test assumes no VSCODE_* environment variables are set @@ -104,7 +106,7 @@ public async Task ApplyAsync_CreatesVsCodeFolderIfNotExists() var vsCodePath = Path.Combine(workspace.WorkspaceRoot.FullName, ".vscode"); var vsCodeCliRunner = new FakeVsCodeCliRunner(null); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); // First, make the scanner find a parent .vscode folder to get an applicator var parentVsCode = workspace.CreateDirectory(".vscode"); @@ -112,7 +114,7 @@ public async Task ApplyAsync_CreatesVsCodeFolderIfNotExists() await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); - // Scanner adds applicators for: Aspire MCP, Playwright MCP, and agent instructions + // Scanner adds applicators for: Aspire MCP, Playwright CLI, and agent instructions Assert.NotEmpty(context.Applicators); var aspireApplicator = context.Applicators.First(a => a.Description.Contains("Aspire MCP")); @@ -131,7 +133,7 @@ public async Task ApplyAsync_CreatesMcpJsonWithCorrectConfiguration() var vsCodeFolder = workspace.CreateDirectory(".vscode"); var vsCodeCliRunner = new FakeVsCodeCliRunner(null); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -187,7 +189,7 @@ public async Task ApplyAsync_PreservesExistingMcpJsonContent() var vsCodeCliRunner = new FakeVsCodeCliRunner(null); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -228,12 +230,12 @@ public async Task ApplyAsync_UpdatesExistingAspireServerConfig() var vsCodeCliRunner = new FakeVsCodeCliRunner(null); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); - // Should return applicators for Aspire MCP, Playwright MCP, and agent instructions + // Should return applicators for Aspire MCP, Playwright CLI, and agent instructions Assert.NotEmpty(context.Applicators); var aspireApplicator = context.Applicators.First(a => a.Description.Contains("Aspire MCP")); @@ -253,39 +255,19 @@ public async Task ApplyAsync_UpdatesExistingAspireServerConfig() } [Fact] - public async Task ApplyAsync_WithConfigurePlaywrightTrue_AddsPlaywrightServer() + public async Task ScanAsync_AddsPlaywrightCliApplicator() { using var workspace = TemporaryWorkspace.Create(outputHelper); var vsCodeFolder = workspace.CreateDirectory(".vscode"); var vsCodeCliRunner = new FakeVsCodeCliRunner(null); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); - - // Apply both MCP-related applicators (Aspire and Playwright) - var aspireApplicator = context.Applicators.First(a => a.Description.Contains("Aspire MCP")); - var playwrightApplicator = context.Applicators.First(a => a.Description.Contains("Playwright MCP")); - await aspireApplicator.ApplyAsync(CancellationToken.None).DefaultTimeout(); - await playwrightApplicator.ApplyAsync(CancellationToken.None).DefaultTimeout(); - - var mcpJsonPath = Path.Combine(vsCodeFolder.FullName, "mcp.json"); - var content = await File.ReadAllTextAsync(mcpJsonPath); - var config = JsonNode.Parse(content)?.AsObject(); - Assert.NotNull(config); - var servers = config["servers"]?.AsObject(); - Assert.NotNull(servers); - - // Both aspire and playwright servers should exist - Assert.True(servers.ContainsKey("aspire")); - Assert.True(servers.ContainsKey("playwright")); - - var playwrightServer = servers["playwright"]?.AsObject(); - Assert.NotNull(playwrightServer); - Assert.Equal("stdio", playwrightServer["type"]?.GetValue()); - Assert.Equal("npx", playwrightServer["command"]?.GetValue()); + // Should have a Playwright CLI installation applicator + Assert.Contains(context.Applicators, a => a.Description.Contains("Playwright CLI")); } [Fact] @@ -300,7 +282,7 @@ public async Task ApplyAsync_WithMalformedMcpJson_ThrowsInvalidOperationExceptio var vsCodeCliRunner = new FakeVsCodeCliRunner(null); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -328,7 +310,7 @@ public async Task ApplyAsync_WithEmptyMcpJson_ThrowsInvalidOperationException() var vsCodeCliRunner = new FakeVsCodeCliRunner(null); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -354,7 +336,7 @@ public async Task ApplyAsync_WithMalformedMcpJson_DoesNotOverwriteFile() var vsCodeCliRunner = new FakeVsCodeCliRunner(null); var executionContext = CreateExecutionContext(workspace.WorkspaceRoot); - var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, executionContext, NullLogger.Instance); + var scanner = new VsCodeAgentEnvironmentScanner(vsCodeCliRunner, CreatePlaywrightCliInstaller(), executionContext, NullLogger.Instance); var context = CreateScanContext(workspace.WorkspaceRoot); await scanner.ScanAsync(context, CancellationToken.None).DefaultTimeout(); @@ -378,6 +360,14 @@ private sealed class FakeVsCodeCliRunner(SemVersion? version) : IVsCodeCliRunner public Task GetVersionAsync(VsCodeRunOptions options, CancellationToken cancellationToken) => Task.FromResult(version); } + private static PlaywrightCliInstaller CreatePlaywrightCliInstaller() + { + return new PlaywrightCliInstaller( + new FakeNpmRunner(), + new FakePlaywrightCliRunner(), + NullLogger.Instance); + } + private static AgentEnvironmentScanContext CreateScanContext( DirectoryInfo workingDirectory, DirectoryInfo? repositoryRoot = null) diff --git a/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs b/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs new file mode 100644 index 00000000000..652b985a603 --- /dev/null +++ b/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Agents.Playwright; +using Aspire.Cli.Npm; +using Semver; + +namespace Aspire.Cli.Tests.TestServices; + +/// +/// A fake implementation of for testing. +/// +internal sealed class FakeNpmRunner : INpmRunner +{ + public Task ResolvePackageAsync(string packageName, string versionRange, CancellationToken cancellationToken) + => Task.FromResult(null); + + public Task PackAsync(string packageName, string version, string outputDirectory, CancellationToken cancellationToken) + => Task.FromResult(null); + + public Task AuditSignaturesAsync(CancellationToken cancellationToken) + => Task.FromResult(true); + + public Task InstallGlobalAsync(string tarballPath, CancellationToken cancellationToken) + => Task.FromResult(true); +} + +/// +/// A fake implementation of for testing. +/// +internal sealed class FakePlaywrightCliRunner : IPlaywrightCliRunner +{ + public Task GetVersionAsync(CancellationToken cancellationToken) + => Task.FromResult(null); + + public Task InstallSkillsAsync(CancellationToken cancellationToken) + => Task.FromResult(true); +} From 6dffb0efc9ba80e144509a452e6da1bd36a359a4 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 24 Feb 2026 13:58:36 +1100 Subject: [PATCH 02/15] Pin Playwright CLI version range to 0.1.1 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs index bad336ad486..5a1c28a21ee 100644 --- a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs +++ b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs @@ -24,7 +24,7 @@ internal sealed class PlaywrightCliInstaller( /// /// The version range to resolve. Updated periodically with Aspire releases. /// - internal const string VersionRange = "0.1"; + internal const string VersionRange = "0.1.1"; /// /// Installs the Playwright CLI with supply chain verification and generates skill files. From e91ce3a0a4612fd15b081d50eb7e7dc650ad1b95 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 26 Feb 2026 15:27:19 +1100 Subject: [PATCH 03/15] Add npm provenance verification and break-glass config - Add INpmProvenanceChecker/NpmProvenanceChecker for SLSA attestation verification - Return rich ProvenanceVerificationResult with gate-specific outcome enum - Fix AuditSignaturesAsync with temp-project approach for global tools - Add disablePlaywrightCliPackageValidation break-glass config option - Add security design document (docs/specs/safe-npm-tool-install.md) - Verify SRI integrity, Sigstore attestations, and source repository provenance Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/specs/safe-npm-tool-install.md | 190 ++++++++++++++++++ .../Playwright/PlaywrightCliInstaller.cs | 86 +++++++- src/Aspire.Cli/Npm/INpmProvenanceChecker.cs | 110 ++++++++++ src/Aspire.Cli/Npm/INpmRunner.cs | 9 +- src/Aspire.Cli/Npm/NpmProvenanceChecker.cs | 169 ++++++++++++++++ src/Aspire.Cli/Npm/NpmRunner.cs | 103 +++++++++- src/Aspire.Cli/Program.cs | 1 + .../ClaudeCodeAgentEnvironmentScannerTests.cs | 3 + .../CopilotCliAgentEnvironmentScannerTests.cs | 3 + .../Agents/NpmProvenanceCheckerTests.cs | 183 +++++++++++++++++ .../OpenCodeAgentEnvironmentScannerTests.cs | 3 + .../Agents/PlaywrightCliInstallerTests.cs | 117 ++++++++++- .../VsCodeAgentEnvironmentScannerTests.cs | 3 + .../TestServices/FakePlaywrightServices.cs | 15 +- 14 files changed, 966 insertions(+), 29 deletions(-) create mode 100644 docs/specs/safe-npm-tool-install.md create mode 100644 src/Aspire.Cli/Npm/INpmProvenanceChecker.cs create mode 100644 src/Aspire.Cli/Npm/NpmProvenanceChecker.cs create mode 100644 tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs diff --git a/docs/specs/safe-npm-tool-install.md b/docs/specs/safe-npm-tool-install.md new file mode 100644 index 00000000000..b6826b9b900 --- /dev/null +++ b/docs/specs/safe-npm-tool-install.md @@ -0,0 +1,190 @@ +# Safe npm Global Tool Installation + +## Overview + +The Aspire CLI installs the `@playwright/cli` npm package as a global tool during `aspire agent init`. Because this tool runs with the user's full privileges, we must verify its authenticity and provenance before installation. This document describes the verification process, the threat model, and the reasoning behind each step. + +## Threat Model + +### What we're protecting against + +1. **Registry compromise** — An attacker gains write access to the npm registry and publishes a malicious version of `@playwright/cli` +2. **Publish token theft** — An attacker steals a maintainer's npm publish token and publishes a tampered package +3. **Man-in-the-middle** — An attacker intercepts the network request and substitutes a different tarball +4. **Dependency confusion** — A malicious package with a similar name is installed instead of the intended one + +### What we're NOT protecting against + +- Compromise of the legitimate source repository (`microsoft/playwright-cli`) itself +- Compromise of the GitHub Actions build infrastructure (Sigstore OIDC provider) +- Compromise of the Sigstore transparency log infrastructure +- Malicious code introduced through legitimate dependencies of `@playwright/cli` + +### Trust anchors + +Our verification chain relies on these trust anchors: + +| Trust anchor | What it provides | How it's protected | +|---|---|---| +| **npm registry** | Package metadata, tarball hosting | HTTPS/TLS, npm's infrastructure security | +| **Sigstore (Fulcio + Rekor)** | Cryptographic attestation signatures | Public CA with OIDC federation, append-only transparency log | +| **GitHub Actions OIDC** | Builder identity claims in Sigstore certificates | GitHub's infrastructure security | +| **Hardcoded expected values** | Package name, version range, expected source repository | Code review, our own release process | + +## Verification Process + +### Step 1: Resolve package version and metadata + +**Action:** Run `npm view @playwright/cli@{versionRange} version` and `npm view @playwright/cli@{version} dist.integrity` to get the resolved version and the registry's SRI integrity hash. + +**What this establishes:** We know the exact version we intend to install and the hash the registry claims for its tarball. + +**Trust basis:** npm registry over HTTPS/TLS. + +**Limitations:** If the registry is compromised, both the version and hash could be attacker-controlled. This step alone is insufficient — it only establishes what the registry *claims*. + +### Step 2: Check if already installed at a suitable version + +**Action:** Run `playwright-cli --version` and compare against the resolved version. + +**What this establishes:** Whether installation can be skipped entirely (already up-to-date or newer). + +**Trust basis:** The previously-installed binary. If the user's system is compromised, this could be spoofed, but that's outside our threat model. + +### Step 3: Verify Sigstore attestations via npm + +**Action:** +1. Create a temporary directory with a minimal `package.json` +2. Run `npm install @playwright/cli@{version} --ignore-scripts` to install the package from the registry as a project dependency +3. Run `npm audit signatures` to verify Sigstore attestation signatures + +**What this establishes:** That valid Sigstore-signed attestations exist for `@playwright/cli@{version}`. Specifically: + +- The npm registry has attestation bundles for this package version +- The attestation signatures are cryptographically valid (signed by Sigstore's Fulcio CA) +- The attestation entries are present in the Rekor transparency log (inclusion proof verified) +- The OIDC identity in the signing certificate corresponds to a GitHub Actions workflow + +**Trust basis:** Sigstore's public key infrastructure. Even if the npm registry is compromised, an attacker cannot forge valid Sigstore signatures — they would need to compromise Fulcio (the Sigstore CA) or obtain a valid OIDC token from GitHub Actions for the legitimate repository's workflow. + +**Why a temporary project is needed:** `npm audit signatures` operates on installed project dependencies. It requires `node_modules` and a `package-lock.json` to know which packages to verify. For a global tool install there is no project context, so we create one temporarily. The package must be installed from the registry (not from a local tarball) because `npm audit signatures` skips packages with `resolved: file:...` in the lockfile. + +**Limitations:** `npm audit signatures` verifies that *valid attestations exist* but does not expose the attestation *content*. It confirms "this package has authentic Sigstore-signed attestations" but does not tell us *what* those attestations say (e.g., which repository built the package). That's addressed in Step 4. + +### Step 4: Verify provenance source repository + +**Action:** +1. Fetch the attestation bundle from `https://registry.npmjs.org/-/npm/v1/attestations/@playwright/cli@{version}` +2. Find the attestation with `predicateType: "https://slsa.dev/provenance/v1"` (SLSA Build L3 provenance) +3. Base64-decode the DSSE envelope payload to extract the in-toto statement +4. Parse `predicate.buildDefinition.externalParameters.workflow.repository` +5. Verify it equals `https://github.com/microsoft/playwright-cli` + +**What this establishes:** That the Sigstore-attested provenance for this package version claims it was built from the `microsoft/playwright-cli` GitHub repository via a GitHub Actions workflow. + +**Trust basis:** The attestation content is protected by the Sigstore signature verified in Step 3. Since Step 3 confirmed the attestation is cryptographically authentic (signed by a valid Sigstore certificate corresponding to a GitHub Actions OIDC identity), the content we read in Step 4 cannot have been tampered with by the npm registry. An attacker would need to compromise Sigstore itself to forge attestation content pointing to `microsoft/playwright-cli`. + +**Why we fetch from the registry API:** The npm CLI (`npm audit signatures`) verifies Sigstore signatures but does not expose provenance content in its output. The `--json` flag only produces `{"invalid":[],"missing":[]}`. There is no npm CLI command to read the source repository from a SLSA provenance attestation. We must fetch the attestation bundle directly from the registry API. + +**Note on reading attested content from an untrusted source:** We are reading the attestation JSON from the same npm registry that could theoretically be compromised. However, the attestation *signature* was already verified by `npm audit signatures` in Step 3 using Sigstore's independent trust chain. We are relying on the fact that the registry serves the same attestation bundle that npm verified. If the registry served different attestation data to our HTTP request than what `npm audit signatures` verified, the provenance content could be spoofed. This is a residual risk — see "Residual Risks" below. + +### Step 5: Download and verify tarball integrity + +**Action:** +1. Run `npm pack @playwright/cli@{version}` to download the tarball +2. Compute SHA-512 hash of the downloaded tarball +3. Compare against the SRI integrity hash obtained in Step 1 + +**What this establishes:** That the tarball we have on disk is bit-for-bit identical to what the npm registry published for this version. + +**Trust basis:** Cryptographic hash comparison (SHA-512). If the hash matches, the content is the same regardless of how it was delivered. + +**Relationship to Step 3:** The Sigstore attestations verified in Step 3 are bound to the package version and its published content. The integrity hash in the registry packument is the canonical identifier for the tarball content. By verifying our tarball matches this hash, we establish that our tarball is the same artifact that the Sigstore attestations cover. + +### Step 6: Install globally from verified tarball + +**Action:** Run `npm install -g {tarballPath}` to install the verified tarball as a global tool. + +**What this establishes:** The tool is installed and available on the user's PATH. + +**Trust basis:** All preceding verification steps have passed. The tarball content has been verified against the registry's published hash (Step 5), the Sigstore attestations for that content are cryptographically valid (Step 3), and the attestations claim the correct source repository (Step 4). + +### Step 7: Generate skill files + +**Action:** Run `playwright-cli install --skills` to generate agent skill files. + +**What this establishes:** The Playwright CLI skill files are available for agent environments. + +## Verification Chain Summary + +``` + ┌─────────────────────────────┐ + │ Hardcoded expectations │ + │ • Package: @playwright/cli │ + │ • Version range: 0.1.1 │ + │ • Source: microsoft/ │ + │ playwright-cli │ + └──────────────┬────────────────┘ + │ + ┌──────────────▼────────────────┐ + │ Step 1: Resolve version + │ + │ integrity hash from registry │ + └──────────────┬────────────────┘ + │ + ┌────────────────────┼────────────────────┐ + │ │ │ + ┌──────────▼──────────┐ ┌─────▼──────────┐ ┌──────▼──────────┐ + │ Step 3: npm audit │ │ Step 4: Verify │ │ Step 5: npm pack│ + │ signatures │ │ provenance repo │ │ + SHA-512 check │ + │ (Sigstore crypto) │ │ (attestation │ │ (tarball │ + │ │ │ content) │ │ integrity) │ + └──────────┬───────────┘ └─────┬──────────┘ └──────┬──────────┘ + │ │ │ + │ Attestation is │ Built from │ Tarball matches + │ authentic │ expected repo │ published hash + └────────────────────┼─────────────────────┘ + │ + ┌──────────────▼────────────────┐ + │ Step 6: npm install -g │ + │ (from verified tarball) │ + └───────────────────────────────┘ +``` + +## Residual Risks + +### 1. Registry serving different attestation data to different clients + +**Risk:** The npm registry could theoretically serve one attestation bundle to `npm audit signatures` (which passes verification) and a different bundle to our HTTP API request (with spoofed provenance content). + +**Mitigation:** This would require active, targeted compromise of the npm registry's serving infrastructure — not just a publish token theft or package tampering. The Rekor transparency log provides a public record of all attestations, making such targeted serving detectable. + +**Alternative mitigation:** We could eliminate this risk entirely by parsing the attestation bundle ourselves and verifying the Sigstore signature directly in C#. This would require implementing ECDSA signature verification, X.509 certificate chain validation, and Merkle inclusion proof verification. This significantly increases implementation complexity and is not recommended for the initial implementation. + +### 2. Time-of-check-to-time-of-use (TOCTOU) + +**Risk:** The package could be replaced on the registry between our verification steps and the global install. + +**Mitigation:** We verify the SHA-512 hash of the tarball we actually install (Step 5), and we install from the local tarball file (not from the registry again). The verified tarball is the same file that gets installed. + +### 3. Transitive dependency attacks + +**Risk:** `@playwright/cli` has dependencies that could be compromised. + +**Mitigation:** The `--ignore-scripts` flag prevents execution of install scripts. However, the dependencies' code runs when the tool is invoked. This is partially mitigated by Sigstore attestations covering the dependency tree, but comprehensive supply chain verification of all transitive dependencies is out of scope. + +## Implementation Constants + +```csharp +internal const string PackageName = "@playwright/cli"; +internal const string VersionRange = "0.1.1"; +internal const string ExpectedSourceRepository = "https://github.com/microsoft/playwright-cli"; +internal const string NpmRegistryAttestationsUrl = "https://registry.npmjs.org/-/npm/v1/attestations"; +internal const string SlsaProvenancePredicateType = "https://slsa.dev/provenance/v1"; +``` + +## Future Improvements + +1. **Direct Sigstore verification in C#** — Eliminate the dependency on `npm audit signatures` by implementing Sigstore bundle verification natively. This would remove residual risk #1 and reduce the dependency on npm CLI. +2. **Rekor log verification** — Independently verify the Rekor inclusion proof to confirm the attestation was logged in the public transparency log. +3. **Certificate identity verification** — Check the OIDC identity claims in the Sigstore signing certificate (issuer, subject, workflow reference) rather than just the provenance payload. +4. **Pinned tarball hash** — Ship a known-good SRI hash with each Aspire release, eliminating the need to trust the registry for the hash at all. diff --git a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs index 5a1c28a21ee..ed1f9dce263 100644 --- a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs +++ b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs @@ -3,6 +3,7 @@ using System.Security.Cryptography; using Aspire.Cli.Npm; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Semver; @@ -13,7 +14,9 @@ namespace Aspire.Cli.Agents.Playwright; /// internal sealed class PlaywrightCliInstaller( INpmRunner npmRunner, + INpmProvenanceChecker provenanceChecker, IPlaywrightCliRunner playwrightCliRunner, + IConfiguration configuration, ILogger logger) { /// @@ -26,6 +29,17 @@ internal sealed class PlaywrightCliInstaller( /// internal const string VersionRange = "0.1.1"; + /// + /// The expected source repository for provenance verification. + /// + internal const string ExpectedSourceRepository = "https://github.com/microsoft/playwright-cli"; + + /// + /// Configuration key that disables package validation when set to "true". + /// This is a break-glass mechanism for debugging npm service issues and must never be the default. + /// + internal const string DisablePackageValidationKey = "disablePlaywrightCliPackageValidation"; + /// /// Installs the Playwright CLI with supply chain verification and generates skill files. /// @@ -67,7 +81,61 @@ public async Task InstallAsync(CancellationToken cancellationToken) packageInfo.Version); } - // Step 3: Download the tarball via npm pack. + // Check break-glass configuration to bypass package validation. + var validationDisabled = string.Equals(configuration[DisablePackageValidationKey], "true", StringComparison.OrdinalIgnoreCase); + if (validationDisabled) + { + logger.LogWarning( + "Package validation is disabled via '{ConfigKey}'. " + + "Sigstore attestation, provenance, and integrity checks will be skipped. " + + "This should only be used for debugging npm service issues.", + DisablePackageValidationKey); + } + + if (!validationDisabled) + { + // Step 3: Verify Sigstore attestation signatures via npm audit signatures. + // This creates a temporary project to give npm audit signatures a working context. + logger.LogDebug("Verifying Sigstore attestations for {Package}@{Version}", PackageName, packageInfo.Version); + var auditPassed = await npmRunner.AuditSignaturesAsync(PackageName, packageInfo.Version.ToString(), cancellationToken); + if (!auditPassed) + { + logger.LogWarning( + "Sigstore attestation verification failed for {Package}@{Version}. The package may not have valid attestations.", + PackageName, + packageInfo.Version); + return false; + } + + logger.LogDebug("Sigstore attestation verification passed for {Package}@{Version}", PackageName, packageInfo.Version); + + // Step 4: Verify provenance source repository from SLSA attestation. + logger.LogDebug("Verifying provenance for {Package}@{Version}", PackageName, packageInfo.Version); + var provenanceResult = await provenanceChecker.VerifyProvenanceAsync( + PackageName, + packageInfo.Version.ToString(), + ExpectedSourceRepository, + cancellationToken); + + if (!provenanceResult.IsVerified) + { + logger.LogWarning( + "Provenance verification failed for {Package}@{Version}: {Outcome}. Expected source repository: {ExpectedRepo}", + PackageName, + packageInfo.Version, + provenanceResult.Outcome, + ExpectedSourceRepository); + return false; + } + + logger.LogDebug( + "Provenance verification passed for {Package}@{Version} (source: {SourceRepo})", + PackageName, + packageInfo.Version, + provenanceResult.Provenance?.SourceRepository); + } + + // Step 5: Download the tarball via npm pack. var tempDir = Path.Combine(Path.GetTempPath(), $"aspire-playwright-{Guid.NewGuid():N}"); Directory.CreateDirectory(tempDir); @@ -82,8 +150,8 @@ public async Task InstallAsync(CancellationToken cancellationToken) return false; } - // Step 4: Verify the downloaded tarball's SHA-512 hash matches the SRI integrity value. - if (!VerifyIntegrity(tarballPath, packageInfo.Integrity)) + // Step 6: Verify the downloaded tarball's SHA-512 hash matches the SRI integrity value. + if (!validationDisabled && !VerifyIntegrity(tarballPath, packageInfo.Integrity)) { logger.LogWarning( "Integrity verification failed for {Package}@{Version}. The downloaded package may have been tampered with.", @@ -92,16 +160,12 @@ public async Task InstallAsync(CancellationToken cancellationToken) return false; } - logger.LogDebug("Integrity verification passed for {TarballPath}", tarballPath); - - // Step 5: Run npm audit signatures for additional provenance verification. - var auditPassed = await npmRunner.AuditSignaturesAsync(cancellationToken); - if (!auditPassed) + if (!validationDisabled) { - logger.LogDebug("npm audit signatures did not pass, continuing with installation"); + logger.LogDebug("Integrity verification passed for {TarballPath}", tarballPath); } - // Step 6: Install globally from the verified tarball. + // Step 7: Install globally from the verified tarball. logger.LogDebug("Installing {Package}@{Version} globally", PackageName, packageInfo.Version); var installSuccess = await npmRunner.InstallGlobalAsync(tarballPath, cancellationToken); @@ -111,7 +175,7 @@ public async Task InstallAsync(CancellationToken cancellationToken) return false; } - // Step 7: Generate skill files. + // Step 8: Generate skill files. logger.LogDebug("Generating Playwright CLI skill files"); return await playwrightCliRunner.InstallSkillsAsync(cancellationToken); } diff --git a/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs b/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs new file mode 100644 index 00000000000..161b9a02fcc --- /dev/null +++ b/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs @@ -0,0 +1,110 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Aspire.Cli.Npm; + +/// +/// Represents the outcome of a provenance verification check. +/// Each value corresponds to a specific gate in the verification process. +/// +internal enum ProvenanceVerificationOutcome +{ + /// + /// All checks passed and the source repository matches the expected value. + /// + Verified, + + /// + /// Failed to fetch attestation data from the npm registry (network error or non-success HTTP status). + /// + AttestationFetchFailed, + + /// + /// The attestation response could not be parsed as valid JSON. + /// + AttestationParseFailed, + + /// + /// No SLSA provenance attestation was found in the registry response. + /// + SlsaProvenanceNotFound, + + /// + /// The DSSE envelope payload could not be decoded from the attestation bundle. + /// + PayloadDecodeFailed, + + /// + /// The source repository could not be extracted from the provenance statement. + /// + SourceRepositoryNotFound, + + /// + /// The attested source repository does not match the expected value. + /// + SourceRepositoryMismatch +} + +/// +/// Represents the deserialized provenance data extracted from an SLSA attestation. +/// +internal sealed class NpmProvenanceData +{ + /// + /// Gets the source repository URL from the attestation (e.g., "https://github.com/microsoft/playwright-cli"). + /// + public string? SourceRepository { get; init; } + + /// + /// Gets the workflow file path from the attestation (e.g., ".github/workflows/publish.yml"). + /// + public string? WorkflowPath { get; init; } + + /// + /// Gets the builder ID URI from the attestation (e.g., "https://github.com/actions/runner/github-hosted"). + /// + public string? BuilderId { get; init; } + + /// + /// Gets the workflow reference (e.g., "refs/tags/v0.1.1"). + /// + public string? WorkflowRef { get; init; } +} + +/// +/// Represents the result of a provenance verification check. +/// +internal sealed class ProvenanceVerificationResult +{ + /// + /// Gets the outcome of the verification, indicating which gate passed or failed. + /// + public required ProvenanceVerificationOutcome Outcome { get; init; } + + /// + /// Gets the deserialized provenance data, if available. May be partially populated + /// depending on how far verification progressed before failure. + /// + public NpmProvenanceData? Provenance { get; init; } + + /// + /// Gets a value indicating whether the verification succeeded. + /// + public bool IsVerified => Outcome is ProvenanceVerificationOutcome.Verified; +} + +/// +/// Verifies npm package provenance by checking SLSA attestations from the npm registry. +/// +internal interface INpmProvenanceChecker +{ + /// + /// Verifies that the SLSA provenance attestation for a package was built from the expected source repository. + /// + /// The npm package name (e.g., "@playwright/cli"). + /// The exact version to verify. + /// The expected source repository URL (e.g., "https://github.com/microsoft/playwright-cli"). + /// A token to cancel the operation. + /// A indicating the outcome and any extracted provenance data. + Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, CancellationToken cancellationToken); +} diff --git a/src/Aspire.Cli/Npm/INpmRunner.cs b/src/Aspire.Cli/Npm/INpmRunner.cs index 582f40ffa70..7bd3eb27934 100644 --- a/src/Aspire.Cli/Npm/INpmRunner.cs +++ b/src/Aspire.Cli/Npm/INpmRunner.cs @@ -46,11 +46,16 @@ internal interface INpmRunner Task PackAsync(string packageName, string version, string outputDirectory, CancellationToken cancellationToken); /// - /// Runs npm audit signatures to verify package provenance. + /// Verifies Sigstore attestation signatures for a package by installing it into a temporary + /// project and running npm audit signatures. This is necessary because npm audit signatures + /// requires a project context (node_modules + package-lock.json) that doesn't exist for + /// global tool installations. /// + /// The npm package name to verify (e.g., "@playwright/cli"). + /// The exact version to verify. /// A token to cancel the operation. /// True if the audit passed, false otherwise. - Task AuditSignaturesAsync(CancellationToken cancellationToken); + Task AuditSignaturesAsync(string packageName, string version, CancellationToken cancellationToken); /// /// Installs a package globally from a local tarball file. diff --git a/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs b/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs new file mode 100644 index 00000000000..fee0fb78b41 --- /dev/null +++ b/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs @@ -0,0 +1,169 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.Logging; + +namespace Aspire.Cli.Npm; + +/// +/// Verifies npm package provenance by fetching and parsing SLSA attestations from the npm registry API. +/// +internal sealed class NpmProvenanceChecker(HttpClient httpClient, ILogger logger) : INpmProvenanceChecker +{ + internal const string NpmRegistryAttestationsBaseUrl = "https://registry.npmjs.org/-/npm/v1/attestations"; + internal const string SlsaProvenancePredicateType = "https://slsa.dev/provenance/v1"; + + /// + public async Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, CancellationToken cancellationToken) + { + // Gate 1: Fetch attestations from the npm registry. + string json; + try + { + var encodedPackage = Uri.EscapeDataString(packageName); + var url = $"{NpmRegistryAttestationsBaseUrl}/{encodedPackage}@{version}"; + + logger.LogDebug("Fetching attestations from {Url}", url); + var response = await httpClient.GetAsync(url, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + logger.LogDebug("Failed to fetch attestations: HTTP {StatusCode}", response.StatusCode); + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationFetchFailed }; + } + + json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException ex) + { + logger.LogDebug(ex, "Failed to fetch attestations for {Package}@{Version}", packageName, version); + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationFetchFailed }; + } + + // Gate 2: Parse the attestation JSON and extract provenance data. + NpmProvenanceData provenance; + try + { + var parseResult = ParseProvenance(json); + if (parseResult is null) + { + return new ProvenanceVerificationResult { Outcome = parseResult?.Outcome ?? ProvenanceVerificationOutcome.SlsaProvenanceNotFound }; + } + + provenance = parseResult.Value.Provenance; + if (parseResult.Value.Outcome is not ProvenanceVerificationOutcome.Verified) + { + return new ProvenanceVerificationResult + { + Outcome = parseResult.Value.Outcome, + Provenance = provenance + }; + } + } + catch (JsonException ex) + { + logger.LogDebug(ex, "Failed to parse attestation response for {Package}@{Version}", packageName, version); + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationParseFailed }; + } + + logger.LogDebug("SLSA provenance source repository: {SourceRepository}", provenance.SourceRepository); + + // Gate 3: Verify the source repository matches. + if (!string.Equals(provenance.SourceRepository, expectedSourceRepository, StringComparison.OrdinalIgnoreCase)) + { + logger.LogWarning( + "Provenance verification failed: expected source repository {Expected} but attestation says {Actual}", + expectedSourceRepository, + provenance.SourceRepository); + + return new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.SourceRepositoryMismatch, + Provenance = provenance + }; + } + + return new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.Verified, + Provenance = provenance + }; + } + + /// + /// Parses provenance data from the npm attestation API response. + /// + internal static (NpmProvenanceData Provenance, ProvenanceVerificationOutcome Outcome)? ParseProvenance(string attestationJson) + { + var doc = JsonNode.Parse(attestationJson); + var attestations = doc?["attestations"]?.AsArray(); + + if (attestations is null || attestations.Count == 0) + { + return (new NpmProvenanceData(), ProvenanceVerificationOutcome.SlsaProvenanceNotFound); + } + + foreach (var attestation in attestations) + { + var predicateType = attestation?["predicateType"]?.GetValue(); + if (!string.Equals(predicateType, SlsaProvenancePredicateType, StringComparison.Ordinal)) + { + continue; + } + + // The SLSA provenance is in the DSSE envelope payload, base64-encoded. + var payload = attestation?["bundle"]?["dsseEnvelope"]?["payload"]?.GetValue(); + if (payload is null) + { + return (new NpmProvenanceData(), ProvenanceVerificationOutcome.PayloadDecodeFailed); + } + + byte[] decodedBytes; + try + { + decodedBytes = Convert.FromBase64String(payload); + } + catch (FormatException) + { + return (new NpmProvenanceData(), ProvenanceVerificationOutcome.PayloadDecodeFailed); + } + + var statement = JsonNode.Parse(decodedBytes); + var workflow = statement + ?["predicate"] + ?["buildDefinition"] + ?["externalParameters"] + ?["workflow"]; + + var repository = workflow?["repository"]?.GetValue(); + var workflowPath = workflow?["path"]?.GetValue(); + var workflowRef = workflow?["ref"]?.GetValue(); + + var builderId = statement + ?["predicate"] + ?["runDetails"] + ?["builder"] + ?["id"] + ?.GetValue(); + + var provenance = new NpmProvenanceData + { + SourceRepository = repository, + WorkflowPath = workflowPath, + WorkflowRef = workflowRef, + BuilderId = builderId + }; + + if (repository is null) + { + return (provenance, ProvenanceVerificationOutcome.SourceRepositoryNotFound); + } + + return (provenance, ProvenanceVerificationOutcome.Verified); + } + + return (new NpmProvenanceData(), ProvenanceVerificationOutcome.SlsaProvenanceNotFound); + } +} diff --git a/src/Aspire.Cli/Npm/NpmRunner.cs b/src/Aspire.Cli/Npm/NpmRunner.cs index 65b9ef0fa19..5a3fee4e991 100644 --- a/src/Aspire.Cli/Npm/NpmRunner.cs +++ b/src/Aspire.Cli/Npm/NpmRunner.cs @@ -96,7 +96,7 @@ internal sealed class NpmRunner(ILogger logger) : INpmRunner } /// - public async Task AuditSignaturesAsync(CancellationToken cancellationToken) + public async Task AuditSignaturesAsync(string packageName, string version, CancellationToken cancellationToken) { var npmPath = FindNpmPath(); if (npmPath is null) @@ -104,12 +104,58 @@ public async Task AuditSignaturesAsync(CancellationToken cancellationToken return false; } - var output = await RunNpmCommandAsync( - npmPath, - ["audit", "signatures"], - cancellationToken); + // npm audit signatures requires a project context (node_modules + package-lock.json). + // For global tool installs there is no project, so we create a temporary one. + // The package must be installed from the registry (not a local tarball) because + // npm audit signatures skips packages with "resolved: file:..." in the lockfile. + var tempDir = Path.Combine(Path.GetTempPath(), $"aspire-npm-audit-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); - return output is not null; + try + { + // Create minimal package.json + var packageJson = Path.Combine(tempDir, "package.json"); + await File.WriteAllTextAsync( + packageJson, + """{"name":"aspire-verify","version":"1.0.0","private":true}""", + cancellationToken).ConfigureAwait(false); + + // Install the package from the registry to get proper attestation metadata + var installOutput = await RunNpmCommandInDirectoryAsync( + npmPath, + ["install", $"{packageName}@{version}", "--ignore-scripts"], + tempDir, + cancellationToken); + + if (installOutput is null) + { + logger.LogDebug("Failed to install {Package}@{Version} into temporary project for audit", packageName, version); + return false; + } + + // Run npm audit signatures in the temporary project directory + var auditOutput = await RunNpmCommandInDirectoryAsync( + npmPath, + ["audit", "signatures"], + tempDir, + cancellationToken); + + return auditOutput is not null; + } + finally + { + try + { + if (Directory.Exists(tempDir)) + { + Directory.Delete(tempDir, recursive: true); + } + } + catch (IOException ex) + { + logger.LogDebug(ex, "Failed to clean up temporary audit directory: {TempDir}", tempDir); + } + } } /// @@ -140,6 +186,51 @@ public async Task InstallGlobalAsync(string tarballPath, CancellationToken return npmPath; } + private async Task RunNpmCommandInDirectoryAsync(string npmPath, string[] args, string workingDirectory, CancellationToken cancellationToken) + { + var argsString = string.Join(" ", args); + logger.LogDebug("Running npm {Args} in {WorkingDirectory}", argsString, workingDirectory); + + try + { + var startInfo = new ProcessStartInfo(npmPath) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = workingDirectory + }; + + foreach (var arg in args) + { + startInfo.ArgumentList.Add(arg); + } + + using var process = new Process { StartInfo = startInfo }; + process.Start(); + + var outputTask = process.StandardOutput.ReadToEndAsync(cancellationToken); + var errorTask = process.StandardError.ReadToEndAsync(cancellationToken); + + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + var errorOutput = await errorTask.ConfigureAwait(false); + logger.LogDebug("npm {Args} returned non-zero exit code {ExitCode}: {Error}", argsString, process.ExitCode, errorOutput.Trim()); + return null; + } + + return await outputTask.ConfigureAwait(false); + } + catch (Exception ex) when (ex is InvalidOperationException or System.ComponentModel.Win32Exception) + { + logger.LogDebug(ex, "Failed to run npm {Args}", argsString); + return null; + } + } + private async Task RunNpmCommandAsync(string npmPath, string[] args, CancellationToken cancellationToken) { var argsString = string.Join(" ", args); diff --git a/src/Aspire.Cli/Program.cs b/src/Aspire.Cli/Program.cs index 518b072beb1..01991026ad1 100644 --- a/src/Aspire.Cli/Program.cs +++ b/src/Aspire.Cli/Program.cs @@ -328,6 +328,7 @@ internal static async Task BuildApplicationAsync(string[] args, Dictionar // Npm and Playwright CLI operations. builder.Services.AddSingleton(); + builder.Services.AddHttpClient(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/tests/Aspire.Cli.Tests/Agents/ClaudeCodeAgentEnvironmentScannerTests.cs b/tests/Aspire.Cli.Tests/Agents/ClaudeCodeAgentEnvironmentScannerTests.cs index 1866074d129..efbbe8f0ed5 100644 --- a/tests/Aspire.Cli.Tests/Agents/ClaudeCodeAgentEnvironmentScannerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/ClaudeCodeAgentEnvironmentScannerTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.AspNetCore.InternalTesting; +using Microsoft.Extensions.Configuration; using Aspire.Cli.Agents; using Aspire.Cli.Agents.ClaudeCode; using Aspire.Cli.Agents.Playwright; @@ -124,7 +125,9 @@ private static PlaywrightCliInstaller CreatePlaywrightCliInstaller() { return new PlaywrightCliInstaller( new FakeNpmRunner(), + new FakeNpmProvenanceChecker(), new FakePlaywrightCliRunner(), + new ConfigurationBuilder().Build(), NullLogger.Instance); } diff --git a/tests/Aspire.Cli.Tests/Agents/CopilotCliAgentEnvironmentScannerTests.cs b/tests/Aspire.Cli.Tests/Agents/CopilotCliAgentEnvironmentScannerTests.cs index a0375a2c5e6..5b705e3c50c 100644 --- a/tests/Aspire.Cli.Tests/Agents/CopilotCliAgentEnvironmentScannerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/CopilotCliAgentEnvironmentScannerTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.AspNetCore.InternalTesting; +using Microsoft.Extensions.Configuration; using System.Text.Json.Nodes; using Aspire.Cli.Agents; using Aspire.Cli.Agents.CopilotCli; @@ -326,7 +327,9 @@ private static PlaywrightCliInstaller CreatePlaywrightCliInstaller() { return new PlaywrightCliInstaller( new FakeNpmRunner(), + new FakeNpmProvenanceChecker(), new FakePlaywrightCliRunner(), + new ConfigurationBuilder().Build(), NullLogger.Instance); } } diff --git a/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs b/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs new file mode 100644 index 00000000000..b3e9d95a842 --- /dev/null +++ b/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs @@ -0,0 +1,183 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Aspire.Cli.Npm; + +namespace Aspire.Cli.Tests.Agents; + +public class NpmProvenanceCheckerTests +{ + [Fact] + public void ParseProvenance_WithValidSlsaProvenance_ReturnsVerifiedWithData() + { + var json = BuildAttestationJson("https://github.com/microsoft/playwright-cli"); + + var result = NpmProvenanceChecker.ParseProvenance(json); + + Assert.NotNull(result); + Assert.Equal(ProvenanceVerificationOutcome.Verified, result.Value.Outcome); + Assert.Equal("https://github.com/microsoft/playwright-cli", result.Value.Provenance.SourceRepository); + Assert.Equal(".github/workflows/publish.yml", result.Value.Provenance.WorkflowPath); + } + + [Fact] + public void ParseProvenance_WithDifferentRepository_ReturnsVerifiedWithThatRepository() + { + var json = BuildAttestationJson("https://github.com/attacker/malicious-package"); + + var result = NpmProvenanceChecker.ParseProvenance(json); + + Assert.NotNull(result); + Assert.Equal(ProvenanceVerificationOutcome.Verified, result.Value.Outcome); + Assert.Equal("https://github.com/attacker/malicious-package", result.Value.Provenance.SourceRepository); + } + + [Fact] + public void ParseProvenance_WithNoSlsaPredicate_ReturnsSlsaProvenanceNotFound() + { + var json = """ + { + "attestations": [ + { + "predicateType": "https://github.com/npm/attestation/tree/main/specs/publish/v0.1", + "bundle": { + "dsseEnvelope": { + "payload": "" + } + } + } + ] + } + """; + + var result = NpmProvenanceChecker.ParseProvenance(json); + + Assert.NotNull(result); + Assert.Equal(ProvenanceVerificationOutcome.SlsaProvenanceNotFound, result.Value.Outcome); + } + + [Fact] + public void ParseProvenance_WithEmptyAttestations_ReturnsSlsaProvenanceNotFound() + { + var json = """{"attestations": []}"""; + + var result = NpmProvenanceChecker.ParseProvenance(json); + + Assert.NotNull(result); + Assert.Equal(ProvenanceVerificationOutcome.SlsaProvenanceNotFound, result.Value.Outcome); + } + + [Fact] + public void ParseProvenance_WithMalformedJson_ThrowsException() + { + Assert.ThrowsAny(() => NpmProvenanceChecker.ParseProvenance("not json")); + } + + [Fact] + public void ParseProvenance_WithMissingWorkflowNode_ReturnsSourceRepositoryNotFound() + { + var statement = new JsonObject + { + ["_type"] = "https://in-toto.io/Statement/v1", + ["predicateType"] = "https://slsa.dev/provenance/v1", + ["predicate"] = new JsonObject + { + ["buildDefinition"] = new JsonObject + { + ["externalParameters"] = new JsonObject() + } + } + }; + + var payload = Convert.ToBase64String(Encoding.UTF8.GetBytes(statement.ToJsonString())); + var json = $$""" + { + "attestations": [ + { + "predicateType": "https://slsa.dev/provenance/v1", + "bundle": { + "dsseEnvelope": { + "payload": "{{payload}}" + } + } + } + ] + } + """; + + var result = NpmProvenanceChecker.ParseProvenance(json); + + Assert.NotNull(result); + Assert.Equal(ProvenanceVerificationOutcome.SourceRepositoryNotFound, result.Value.Outcome); + } + + [Fact] + public void ParseProvenance_WithMissingPayload_ReturnsPayloadDecodeFailed() + { + var json = """ + { + "attestations": [ + { + "predicateType": "https://slsa.dev/provenance/v1", + "bundle": { + "dsseEnvelope": {} + } + } + ] + } + """; + + var result = NpmProvenanceChecker.ParseProvenance(json); + + Assert.NotNull(result); + Assert.Equal(ProvenanceVerificationOutcome.PayloadDecodeFailed, result.Value.Outcome); + } + + private static string BuildAttestationJson(string sourceRepository) + { + var statement = new JsonObject + { + ["_type"] = "https://in-toto.io/Statement/v1", + ["predicateType"] = "https://slsa.dev/provenance/v1", + ["predicate"] = new JsonObject + { + ["buildDefinition"] = new JsonObject + { + ["externalParameters"] = new JsonObject + { + ["workflow"] = new JsonObject + { + ["repository"] = sourceRepository, + ["path"] = ".github/workflows/publish.yml" + } + } + } + } + }; + + var payload = Convert.ToBase64String(Encoding.UTF8.GetBytes(statement.ToJsonString())); + + var attestationResponse = new JsonObject + { + ["attestations"] = new JsonArray + { + new JsonObject + { + ["predicateType"] = "https://slsa.dev/provenance/v1", + ["bundle"] = new JsonObject + { + ["dsseEnvelope"] = new JsonObject + { + ["payload"] = payload + } + } + } + } + }; + + return attestationResponse.ToJsonString(); + } +} diff --git a/tests/Aspire.Cli.Tests/Agents/OpenCodeAgentEnvironmentScannerTests.cs b/tests/Aspire.Cli.Tests/Agents/OpenCodeAgentEnvironmentScannerTests.cs index 6e5c40542b1..af64edf373b 100644 --- a/tests/Aspire.Cli.Tests/Agents/OpenCodeAgentEnvironmentScannerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/OpenCodeAgentEnvironmentScannerTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.AspNetCore.InternalTesting; +using Microsoft.Extensions.Configuration; using Aspire.Cli.Agents; using Aspire.Cli.Agents.OpenCode; using Aspire.Cli.Agents.Playwright; @@ -104,7 +105,9 @@ private static PlaywrightCliInstaller CreatePlaywrightCliInstaller() { return new PlaywrightCliInstaller( new FakeNpmRunner(), + new FakeNpmProvenanceChecker(), new FakePlaywrightCliRunner(), + new ConfigurationBuilder().Build(), NullLogger.Instance); } diff --git a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs index a3756ef40bf..e29fb84ab4b 100644 --- a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs @@ -4,6 +4,7 @@ using System.Security.Cryptography; using Aspire.Cli.Agents.Playwright; using Aspire.Cli.Npm; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; using Semver; @@ -19,7 +20,7 @@ public async Task InstallAsync_WhenNpmResolveReturnsNull_ReturnsFalse() ResolveResult = null }; var playwrightRunner = new TestPlaywrightCliRunner(); - var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -39,7 +40,7 @@ public async Task InstallAsync_WhenAlreadyInstalledAtSameVersion_SkipsInstallAnd InstalledVersion = version, InstallSkillsResult = true }; - var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -63,7 +64,7 @@ public async Task InstallAsync_WhenNewerVersionInstalled_SkipsInstallAndInstalls InstalledVersion = installedVersion, InstallSkillsResult = true }; - var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -82,7 +83,7 @@ public async Task InstallAsync_WhenPackFails_ReturnsFalse() PackResult = null }; var playwrightRunner = new TestPlaywrightCliRunner(); - var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -108,7 +109,7 @@ public async Task InstallAsync_WhenIntegrityCheckFails_ReturnsFalse() PackResult = tarballPath }; var playwrightRunner = new TestPlaywrightCliRunner(); - var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -148,7 +149,7 @@ public async Task InstallAsync_WhenIntegrityCheckPasses_InstallsGlobally() { InstallSkillsResult = true }; - var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -184,7 +185,7 @@ public async Task InstallAsync_WhenGlobalInstallFails_ReturnsFalse() InstallGlobalResult = false }; var playwrightRunner = new TestPlaywrightCliRunner(); - var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -224,7 +225,7 @@ public async Task InstallAsync_WhenOlderVersionInstalled_PerformsUpgrade() InstalledVersion = installedVersion, InstallSkillsResult = true }; - var installer = new PlaywrightCliInstaller(npmRunner, playwrightRunner, NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -291,6 +292,86 @@ public void VerifyIntegrity_WithNonSha512Prefix_ReturnsFalse() } } + [Fact] + public async Task InstallAsync_WhenAuditSignaturesFails_ReturnsFalse() + { + var version = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); + var npmRunner = new TestNpmRunner + { + ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-abc123" }, + AuditResult = false + }; + var provenanceChecker = new TestNpmProvenanceChecker(); + var playwrightRunner = new TestPlaywrightCliRunner(); + var installer = new PlaywrightCliInstaller(npmRunner, provenanceChecker, playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + + var result = await installer.InstallAsync(CancellationToken.None); + + Assert.False(result); + Assert.False(provenanceChecker.ProvenanceCalled); + } + + [Fact] + public async Task InstallAsync_WhenProvenanceCheckFails_ReturnsFalse() + { + var version = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); + var npmRunner = new TestNpmRunner + { + ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-abc123" }, + AuditResult = true + }; + var provenanceChecker = new TestNpmProvenanceChecker { ProvenanceOutcome = ProvenanceVerificationOutcome.SourceRepositoryMismatch }; + var playwrightRunner = new TestPlaywrightCliRunner(); + var installer = new PlaywrightCliInstaller(npmRunner, provenanceChecker, playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + + var result = await installer.InstallAsync(CancellationToken.None); + + Assert.False(result); + Assert.True(provenanceChecker.ProvenanceCalled); + Assert.False(npmRunner.PackCalled); + } + + [Fact] + public async Task InstallAsync_WhenValidationDisabled_SkipsAllValidationChecks() + { + var version = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); + var tempDir = Path.Combine(Path.GetTempPath(), $"test-playwright-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + var tarballPath = Path.Combine(tempDir, "package.tgz"); + await File.WriteAllBytesAsync(tarballPath, [10, 20, 30]); + + // Use a mismatched integrity hash — validation is disabled so it should still succeed. + try + { + var npmRunner = new TestNpmRunner + { + ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-wronghash" }, + AuditResult = false, + PackResult = tarballPath + }; + var provenanceChecker = new TestNpmProvenanceChecker { ProvenanceOutcome = ProvenanceVerificationOutcome.AttestationFetchFailed }; + var playwrightRunner = new TestPlaywrightCliRunner { InstallSkillsResult = true }; + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + [PlaywrightCliInstaller.DisablePackageValidationKey] = "true" + }) + .Build(); + var installer = new PlaywrightCliInstaller(npmRunner, provenanceChecker, playwrightRunner, configuration, NullLogger.Instance); + + var result = await installer.InstallAsync(CancellationToken.None); + + Assert.True(result); + Assert.False(provenanceChecker.ProvenanceCalled); + Assert.True(npmRunner.PackCalled); + Assert.True(npmRunner.InstallGlobalCalled); + } + finally + { + Directory.Delete(tempDir, true); + } + } + private sealed class TestNpmRunner : INpmRunner { public NpmPackageInfo? ResolveResult { get; set; } @@ -310,7 +391,7 @@ private sealed class TestNpmRunner : INpmRunner return Task.FromResult(PackResult); } - public Task AuditSignaturesAsync(CancellationToken cancellationToken) + public Task AuditSignaturesAsync(string packageName, string version, CancellationToken cancellationToken) => Task.FromResult(AuditResult); public Task InstallGlobalAsync(string tarballPath, CancellationToken cancellationToken) @@ -320,6 +401,24 @@ public Task InstallGlobalAsync(string tarballPath, CancellationToken cance } } + private sealed class TestNpmProvenanceChecker : INpmProvenanceChecker + { + public ProvenanceVerificationOutcome ProvenanceOutcome { get; set; } = ProvenanceVerificationOutcome.Verified; + public bool ProvenanceCalled { get; private set; } + + public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, CancellationToken cancellationToken) + { + ProvenanceCalled = true; + return Task.FromResult(new ProvenanceVerificationResult + { + Outcome = ProvenanceOutcome, + Provenance = ProvenanceOutcome is ProvenanceVerificationOutcome.Verified + ? new NpmProvenanceData { SourceRepository = expectedSourceRepository } + : new NpmProvenanceData() + }); + } + } + private sealed class TestPlaywrightCliRunner : IPlaywrightCliRunner { public SemVersion? InstalledVersion { get; set; } diff --git a/tests/Aspire.Cli.Tests/Agents/VsCodeAgentEnvironmentScannerTests.cs b/tests/Aspire.Cli.Tests/Agents/VsCodeAgentEnvironmentScannerTests.cs index 92ab5d41b10..65c80be1ebe 100644 --- a/tests/Aspire.Cli.Tests/Agents/VsCodeAgentEnvironmentScannerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/VsCodeAgentEnvironmentScannerTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.AspNetCore.InternalTesting; +using Microsoft.Extensions.Configuration; using System.Text.Json.Nodes; using Aspire.Cli.Agents; using Aspire.Cli.Agents.Playwright; @@ -364,7 +365,9 @@ private static PlaywrightCliInstaller CreatePlaywrightCliInstaller() { return new PlaywrightCliInstaller( new FakeNpmRunner(), + new FakeNpmProvenanceChecker(), new FakePlaywrightCliRunner(), + new ConfigurationBuilder().Build(), NullLogger.Instance); } diff --git a/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs b/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs index 652b985a603..1f80d78af0f 100644 --- a/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs +++ b/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs @@ -18,13 +18,26 @@ internal sealed class FakeNpmRunner : INpmRunner public Task PackAsync(string packageName, string version, string outputDirectory, CancellationToken cancellationToken) => Task.FromResult(null); - public Task AuditSignaturesAsync(CancellationToken cancellationToken) + public Task AuditSignaturesAsync(string packageName, string version, CancellationToken cancellationToken) => Task.FromResult(true); public Task InstallGlobalAsync(string tarballPath, CancellationToken cancellationToken) => Task.FromResult(true); } +/// +/// A fake implementation of for testing. +/// +internal sealed class FakeNpmProvenanceChecker : INpmProvenanceChecker +{ + public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, CancellationToken cancellationToken) + => Task.FromResult(new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.Verified, + Provenance = new NpmProvenanceData { SourceRepository = expectedSourceRepository } + }); +} + /// /// A fake implementation of for testing. /// From abc4b83c91fef99de5d8e366340b7481a96eeb5c Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 26 Feb 2026 15:40:06 +1100 Subject: [PATCH 04/15] Fix markdownlint: add language to fenced code block Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/specs/safe-npm-tool-install.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/specs/safe-npm-tool-install.md b/docs/specs/safe-npm-tool-install.md index b6826b9b900..16893f933e6 100644 --- a/docs/specs/safe-npm-tool-install.md +++ b/docs/specs/safe-npm-tool-install.md @@ -117,8 +117,7 @@ Our verification chain relies on these trust anchors: ## Verification Chain Summary -``` - ┌─────────────────────────────┐ +```text │ Hardcoded expectations │ │ • Package: @playwright/cli │ │ • Version range: 0.1.1 │ From 274de7960c357dffd82c32cdc5af22f56c831450 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 26 Feb 2026 18:49:22 +1100 Subject: [PATCH 05/15] Improve version resolution and provenance verification - Change version range from exact 0.1.1 to >=0.1.1 for future versions - Add playwrightCliVersion config override for pinning specific versions - Verify workflow path (.github/workflows/publish.yml) in provenance - Verify SLSA build type (GitHub Actions) to confirm OIDC token issuer - Add BuildType to NpmProvenanceData, WorkflowMismatch and BuildTypeMismatch outcomes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Playwright/PlaywrightCliInstaller.cs | 35 ++++++++++++-- src/Aspire.Cli/Npm/INpmProvenanceChecker.cs | 27 +++++++++-- src/Aspire.Cli/Npm/NpmProvenanceChecker.cs | 46 ++++++++++++++++--- .../Agents/NpmProvenanceCheckerTests.cs | 14 +++++- .../Agents/PlaywrightCliInstallerTests.cs | 2 +- .../TestServices/FakePlaywrightServices.cs | 2 +- 6 files changed, 108 insertions(+), 18 deletions(-) diff --git a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs index ed1f9dce263..20995cfc1d0 100644 --- a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs +++ b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs @@ -25,21 +25,38 @@ internal sealed class PlaywrightCliInstaller( internal const string PackageName = "@playwright/cli"; /// - /// The version range to resolve. Updated periodically with Aspire releases. + /// The version range to resolve. Accepts any version from 0.1.1 onwards. /// - internal const string VersionRange = "0.1.1"; + internal const string VersionRange = ">=0.1.1"; /// /// The expected source repository for provenance verification. /// internal const string ExpectedSourceRepository = "https://github.com/microsoft/playwright-cli"; + /// + /// The expected workflow file path in the source repository. + /// + internal const string ExpectedWorkflowPath = ".github/workflows/publish.yml"; + + /// + /// The expected SLSA build type, which identifies GitHub Actions as the CI system + /// and implicitly confirms the OIDC token issuer is https://token.actions.githubusercontent.com. + /// + internal const string ExpectedBuildType = "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1"; + /// /// Configuration key that disables package validation when set to "true". /// This is a break-glass mechanism for debugging npm service issues and must never be the default. /// internal const string DisablePackageValidationKey = "disablePlaywrightCliPackageValidation"; + /// + /// Configuration key that overrides the version to install. When set, the specified + /// exact version is used instead of resolving the latest from the version range. + /// + internal const string VersionOverrideKey = "playwrightCliVersion"; + /// /// Installs the Playwright CLI with supply chain verification and generates skill files. /// @@ -48,8 +65,16 @@ internal sealed class PlaywrightCliInstaller( public async Task InstallAsync(CancellationToken cancellationToken) { // Step 1: Resolve the target version and integrity hash from the npm registry. - logger.LogDebug("Resolving {Package}@{Range} from npm registry", PackageName, VersionRange); - var packageInfo = await npmRunner.ResolvePackageAsync(PackageName, VersionRange, cancellationToken); + var versionOverride = configuration[VersionOverrideKey]; + var effectiveRange = !string.IsNullOrEmpty(versionOverride) ? versionOverride : VersionRange; + + if (!string.IsNullOrEmpty(versionOverride)) + { + logger.LogDebug("Using version override from '{ConfigKey}': {Version}", VersionOverrideKey, versionOverride); + } + + logger.LogDebug("Resolving {Package}@{Range} from npm registry", PackageName, effectiveRange); + var packageInfo = await npmRunner.ResolvePackageAsync(PackageName, effectiveRange, cancellationToken); if (packageInfo is null) { @@ -115,6 +140,8 @@ public async Task InstallAsync(CancellationToken cancellationToken) PackageName, packageInfo.Version.ToString(), ExpectedSourceRepository, + ExpectedWorkflowPath, + ExpectedBuildType, cancellationToken); if (!provenanceResult.IsVerified) diff --git a/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs b/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs index 161b9a02fcc..fc0635633fa 100644 --- a/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs +++ b/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs @@ -42,7 +42,18 @@ internal enum ProvenanceVerificationOutcome /// /// The attested source repository does not match the expected value. /// - SourceRepositoryMismatch + SourceRepositoryMismatch, + + /// + /// The attested workflow path does not match the expected value. + /// + WorkflowMismatch, + + /// + /// The SLSA build type does not match the expected GitHub Actions build type, + /// indicating the package was not built by the expected CI system. + /// + BuildTypeMismatch } /// @@ -69,6 +80,13 @@ internal sealed class NpmProvenanceData /// Gets the workflow reference (e.g., "refs/tags/v0.1.1"). /// public string? WorkflowRef { get; init; } + + /// + /// Gets the SLSA build type URI which identifies the CI system used to build the package + /// (e.g., "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1" for GitHub Actions). + /// This implicitly confirms the OIDC token issuer (e.g., https://token.actions.githubusercontent.com). + /// + public string? BuildType { get; init; } } /// @@ -99,12 +117,15 @@ internal sealed class ProvenanceVerificationResult internal interface INpmProvenanceChecker { /// - /// Verifies that the SLSA provenance attestation for a package was built from the expected source repository. + /// Verifies that the SLSA provenance attestation for a package was built from the expected source repository, + /// using the expected workflow, and with the expected build system. /// /// The npm package name (e.g., "@playwright/cli"). /// The exact version to verify. /// The expected source repository URL (e.g., "https://github.com/microsoft/playwright-cli"). + /// The expected workflow file path (e.g., ".github/workflows/publish.yml"). + /// The expected SLSA build type URI identifying the CI system. /// A token to cancel the operation. /// A indicating the outcome and any extracted provenance data. - Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, CancellationToken cancellationToken); + Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, CancellationToken cancellationToken); } diff --git a/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs b/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs index fee0fb78b41..3aca8091f21 100644 --- a/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs +++ b/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs @@ -16,7 +16,7 @@ internal sealed class NpmProvenanceChecker(HttpClient httpClient, ILogger - public async Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, CancellationToken cancellationToken) + public async Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, CancellationToken cancellationToken) { // Gate 1: Fetch attestations from the npm registry. string json; @@ -85,6 +85,36 @@ public async Task VerifyProvenanceAsync(string pac }; } + // Gate 4: Verify the workflow path matches. + if (!string.Equals(provenance.WorkflowPath, expectedWorkflowPath, StringComparison.Ordinal)) + { + logger.LogWarning( + "Provenance verification failed: expected workflow path {Expected} but attestation says {Actual}", + expectedWorkflowPath, + provenance.WorkflowPath); + + return new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.WorkflowMismatch, + Provenance = provenance + }; + } + + // Gate 5: Verify the build type matches (confirms CI system and OIDC token issuer). + if (!string.Equals(provenance.BuildType, expectedBuildType, StringComparison.Ordinal)) + { + logger.LogWarning( + "Provenance verification failed: expected build type {Expected} but attestation says {Actual}", + expectedBuildType, + provenance.BuildType); + + return new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.BuildTypeMismatch, + Provenance = provenance + }; + } + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.Verified, @@ -131,9 +161,9 @@ internal static (NpmProvenanceData Provenance, ProvenanceVerificationOutcome Out } var statement = JsonNode.Parse(decodedBytes); - var workflow = statement - ?["predicate"] - ?["buildDefinition"] + var predicate = statement?["predicate"]; + var buildDefinition = predicate?["buildDefinition"]; + var workflow = buildDefinition ?["externalParameters"] ?["workflow"]; @@ -141,19 +171,21 @@ internal static (NpmProvenanceData Provenance, ProvenanceVerificationOutcome Out var workflowPath = workflow?["path"]?.GetValue(); var workflowRef = workflow?["ref"]?.GetValue(); - var builderId = statement - ?["predicate"] + var builderId = predicate ?["runDetails"] ?["builder"] ?["id"] ?.GetValue(); + var buildType = buildDefinition?["buildType"]?.GetValue(); + var provenance = new NpmProvenanceData { SourceRepository = repository, WorkflowPath = workflowPath, WorkflowRef = workflowRef, - BuilderId = builderId + BuilderId = builderId, + BuildType = buildType }; if (repository is null) diff --git a/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs b/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs index b3e9d95a842..130ff0e0770 100644 --- a/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs @@ -21,6 +21,8 @@ public void ParseProvenance_WithValidSlsaProvenance_ReturnsVerifiedWithData() Assert.Equal(ProvenanceVerificationOutcome.Verified, result.Value.Outcome); Assert.Equal("https://github.com/microsoft/playwright-cli", result.Value.Provenance.SourceRepository); Assert.Equal(".github/workflows/publish.yml", result.Value.Provenance.WorkflowPath); + Assert.Equal("https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", result.Value.Provenance.BuildType); + Assert.Equal("https://github.com/actions/runner/github-hosted", result.Value.Provenance.BuilderId); } [Fact] @@ -136,7 +138,7 @@ public void ParseProvenance_WithMissingPayload_ReturnsPayloadDecodeFailed() Assert.Equal(ProvenanceVerificationOutcome.PayloadDecodeFailed, result.Value.Outcome); } - private static string BuildAttestationJson(string sourceRepository) + private static string BuildAttestationJson(string sourceRepository, string workflowPath = ".github/workflows/publish.yml", string buildType = "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1") { var statement = new JsonObject { @@ -146,14 +148,22 @@ private static string BuildAttestationJson(string sourceRepository) { ["buildDefinition"] = new JsonObject { + ["buildType"] = buildType, ["externalParameters"] = new JsonObject { ["workflow"] = new JsonObject { ["repository"] = sourceRepository, - ["path"] = ".github/workflows/publish.yml" + ["path"] = workflowPath } } + }, + ["runDetails"] = new JsonObject + { + ["builder"] = new JsonObject + { + ["id"] = "https://github.com/actions/runner/github-hosted" + } } } }; diff --git a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs index e29fb84ab4b..fff411e0630 100644 --- a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs @@ -406,7 +406,7 @@ private sealed class TestNpmProvenanceChecker : INpmProvenanceChecker public ProvenanceVerificationOutcome ProvenanceOutcome { get; set; } = ProvenanceVerificationOutcome.Verified; public bool ProvenanceCalled { get; private set; } - public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, CancellationToken cancellationToken) + public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, CancellationToken cancellationToken) { ProvenanceCalled = true; return Task.FromResult(new ProvenanceVerificationResult diff --git a/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs b/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs index 1f80d78af0f..4f8e7d1eb3d 100644 --- a/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs +++ b/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs @@ -30,7 +30,7 @@ public Task InstallGlobalAsync(string tarballPath, CancellationToken cance /// internal sealed class FakeNpmProvenanceChecker : INpmProvenanceChecker { - public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, CancellationToken cancellationToken) + public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, CancellationToken cancellationToken) => Task.FromResult(new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.Verified, From ce6c598311dd75de6d3b38e78761bcccecc601cc Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 26 Feb 2026 19:05:24 +1100 Subject: [PATCH 06/15] Add tests for version pinning and default version range Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Agents/PlaywrightCliInstallerTests.cs | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs index fff411e0630..f4342d18ac9 100644 --- a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs @@ -372,6 +372,44 @@ public async Task InstallAsync_WhenValidationDisabled_SkipsAllValidationChecks() } } + [Fact] + public async Task InstallAsync_WhenVersionOverrideConfigured_UsesOverrideVersion() + { + var version = SemVersion.Parse("0.2.0", SemVersionStyles.Strict); + var npmRunner = new TestNpmRunner + { + ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-abc123" } + }; + var playwrightRunner = new TestPlaywrightCliRunner(); + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + [PlaywrightCliInstaller.VersionOverrideKey] = "0.2.0" + }) + .Build(); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, configuration, NullLogger.Instance); + + await installer.InstallAsync(CancellationToken.None); + + Assert.Equal("0.2.0", npmRunner.ResolvedVersionRange); + } + + [Fact] + public async Task InstallAsync_WhenNoVersionOverride_UsesDefaultRange() + { + var version = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); + var npmRunner = new TestNpmRunner + { + ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-abc123" } + }; + var playwrightRunner = new TestPlaywrightCliRunner(); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + + await installer.InstallAsync(CancellationToken.None); + + Assert.Equal(PlaywrightCliInstaller.VersionRange, npmRunner.ResolvedVersionRange); + } + private sealed class TestNpmRunner : INpmRunner { public NpmPackageInfo? ResolveResult { get; set; } @@ -381,9 +419,13 @@ private sealed class TestNpmRunner : INpmRunner public bool PackCalled { get; private set; } public bool InstallGlobalCalled { get; private set; } + public string? ResolvedVersionRange { get; private set; } public Task ResolvePackageAsync(string packageName, string versionRange, CancellationToken cancellationToken) - => Task.FromResult(ResolveResult); + { + ResolvedVersionRange = versionRange; + return Task.FromResult(ResolveResult); + } public Task PackAsync(string packageName, string version, string outputDirectory, CancellationToken cancellationToken) { From aaf29084a18b7d90fcae956a90797c92cb84b113 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 26 Feb 2026 19:18:26 +1100 Subject: [PATCH 07/15] Add E2E test for Playwright CLI installation via agent init Verifies the full lifecycle: project creation, aspire agent init with Claude Code environment, Playwright CLI installation with npm provenance verification, and skill file generation. Marked as OuterloopTest since it requires npm and network access. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../PlaywrightCliInstallTests.cs | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 tests/Aspire.Cli.EndToEnd.Tests/PlaywrightCliInstallTests.cs diff --git a/tests/Aspire.Cli.EndToEnd.Tests/PlaywrightCliInstallTests.cs b/tests/Aspire.Cli.EndToEnd.Tests/PlaywrightCliInstallTests.cs new file mode 100644 index 00000000000..5e658c25d3f --- /dev/null +++ b/tests/Aspire.Cli.EndToEnd.Tests/PlaywrightCliInstallTests.cs @@ -0,0 +1,160 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.EndToEnd.Tests.Helpers; +using Aspire.Cli.Tests.Utils; +using Hex1b; +using Hex1b.Automation; +using Aspire.TestUtilities; +using Xunit; + +namespace Aspire.Cli.EndToEnd.Tests; + +/// +/// End-to-end test verifying that the Playwright CLI installation flow works correctly +/// through aspire agent init, including npm provenance verification and skill file generation. +/// +[OuterloopTest("Requires npm and network access to install @playwright/cli from the npm registry")] +public sealed class PlaywrightCliInstallTests(ITestOutputHelper output) +{ + /// + /// Verifies the full Playwright CLI installation lifecycle: + /// 1. Playwright CLI is not initially installed + /// 2. An Aspire project is created + /// 3. aspire agent init is run with Claude Code environment selected + /// 4. Playwright CLI is installed and available on PATH + /// 5. The .claude/skills/playwright-cli/SKILL.md skill file is generated + /// + [Fact] + public async Task AgentInit_InstallsPlaywrightCli_AndGeneratesSkillFiles() + { + var workspace = TemporaryWorkspace.Create(output); + + var prNumber = CliE2ETestHelpers.GetRequiredPrNumber(); + var commitSha = CliE2ETestHelpers.GetRequiredCommitSha(); + var isCI = CliE2ETestHelpers.IsRunningInCI; + var recordingPath = CliE2ETestHelpers.GetTestResultsRecordingPath( + nameof(AgentInit_InstallsPlaywrightCli_AndGeneratesSkillFiles)); + + var builder = Hex1bTerminal.CreateBuilder() + .WithHeadless() + .WithDimensions(160, 48) + .WithAsciinemaRecording(recordingPath) + .WithPtyProcess("/bin/bash", ["--norc"]); + + using var terminal = builder.Build(); + + var pendingRun = terminal.RunAsync(TestContext.Current.CancellationToken); + + // Patterns for prompt detection + var workspacePrompt = new CellPatternSearcher().Find("workspace:"); + var agentEnvPrompt = new CellPatternSearcher().Find("agent environments"); + var additionalOptionsPrompt = new CellPatternSearcher().Find("additional options"); + var playwrightOption = new CellPatternSearcher().Find("Install Playwright CLI"); + var configComplete = new CellPatternSearcher().Find("configuration complete"); + var skillFileExists = new CellPatternSearcher().Find("SKILL.md"); + + var counter = new SequenceCounter(); + var sequenceBuilder = new Hex1bTerminalInputSequenceBuilder(); + + sequenceBuilder.PrepareEnvironment(workspace, counter); + + if (isCI) + { + sequenceBuilder.InstallAspireCliFromPullRequest(prNumber, counter); + sequenceBuilder.SourceAspireCliEnvironment(counter); + sequenceBuilder.VerifyAspireCliVersion(commitSha, counter); + } + + // Step 1: Verify playwright-cli is not installed. + sequenceBuilder + .Type("playwright-cli --version 2>&1 || true") + .Enter() + .WaitForSuccessPrompt(counter); + + // Step 2: Create an Aspire project (accept all defaults). + var starterAppTemplate = new CellPatternSearcher().FindPattern("> Starter App"); + var projectNamePrompt = new CellPatternSearcher().Find("Enter the project name"); + var outputPathPrompt = new CellPatternSearcher().Find("Enter the output path"); + var urlsPrompt = new CellPatternSearcher().Find("*.dev.localhost URLs"); + var redisPrompt = new CellPatternSearcher().Find("Use Redis Cache"); + var testProjectPrompt = new CellPatternSearcher().Find("Do you want to create a test project?"); + + sequenceBuilder + .Type("aspire new") + .Enter() + .WaitUntil(s => starterAppTemplate.Search(s).Count > 0, TimeSpan.FromSeconds(60)) + .Enter() // Select Starter App template + .WaitUntil(s => projectNamePrompt.Search(s).Count > 0, TimeSpan.FromSeconds(30)) + .Type("TestProject") + .Enter() + .WaitUntil(s => outputPathPrompt.Search(s).Count > 0, TimeSpan.FromSeconds(10)) + .Enter() // Accept default output path + .WaitUntil(s => urlsPrompt.Search(s).Count > 0, TimeSpan.FromSeconds(10)) + .Enter() // Accept default URL setting + .WaitUntil(s => redisPrompt.Search(s).Count > 0, TimeSpan.FromSeconds(10)) + .Enter() // Accept default Redis setting + .WaitUntil(s => testProjectPrompt.Search(s).Count > 0, TimeSpan.FromSeconds(10)) + .Enter() // Accept default test project setting + .WaitForSuccessPrompt(counter); + + // Step 3: Navigate into the project and create .claude folder to trigger Claude Code detection. + sequenceBuilder + .Type("cd TestProject && mkdir -p .claude") + .Enter() + .WaitForSuccessPrompt(counter); + + // Step 4: Run aspire agent init. + // First prompt: workspace path + sequenceBuilder + .Type("aspire agent init") + .Enter() + .WaitUntil(s => workspacePrompt.Search(s).Count > 0, TimeSpan.FromSeconds(30)) + .Wait(500) + .Enter(); // Accept default workspace path + + // Second prompt: agent environments (select Claude Code) + sequenceBuilder + .WaitUntil(s => agentEnvPrompt.Search(s).Count > 0, TimeSpan.FromSeconds(60)) + .Type(" ") // Toggle first option (Claude Code) + .Enter(); + + // Third prompt: additional options (select Playwright CLI installation) + // Aspire skill file (priority 0) appears first, Playwright CLI (priority 1) second. + sequenceBuilder + .WaitUntil(s => additionalOptionsPrompt.Search(s).Count > 0, TimeSpan.FromSeconds(30)) + .WaitUntil(s => playwrightOption.Search(s).Count > 0, TimeSpan.FromSeconds(10)) + .Type(" ") // Toggle first option (Aspire skill file) + .Key(Hex1b.Input.Hex1bKey.DownArrow) // Move to Playwright CLI option + .Type(" ") // Toggle Playwright CLI option + .Enter(); + + // Wait for installation to complete (this downloads from npm, can take a while) + sequenceBuilder + .WaitUntil(s => configComplete.Search(s).Count > 0, TimeSpan.FromMinutes(3)) + .WaitForSuccessPrompt(counter); + + // Step 5: Verify playwright-cli is now installed. + sequenceBuilder + .Type("playwright-cli --version") + .Enter() + .WaitForSuccessPrompt(counter); + + // Step 6: Verify the skill file was generated. + sequenceBuilder + .Type("ls .claude/skills/playwright-cli/SKILL.md") + .Enter() + .WaitUntil(s => skillFileExists.Search(s).Count > 0, TimeSpan.FromSeconds(10)) + .WaitForSuccessPrompt(counter); + + sequenceBuilder + .Type("exit") + .Enter(); + + var sequence = sequenceBuilder.Build(); + + await sequence.ApplyAsync(terminal, TestContext.Current.CancellationToken); + + await pendingRun; + } +} From acc4286562468258b108e33ff0a5a2391a8b7a82 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 26 Feb 2026 20:11:25 +1100 Subject: [PATCH 08/15] Show status spinner during Playwright CLI installation Wrap the installation work in IInteractionService.ShowStatusAsync to display a spinner with 'Installing Playwright CLI...' status text while the npm operations are in progress. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Playwright/PlaywrightCliInstaller.cs | 9 +++++++ .../ClaudeCodeAgentEnvironmentScannerTests.cs | 1 + .../CopilotCliAgentEnvironmentScannerTests.cs | 1 + .../OpenCodeAgentEnvironmentScannerTests.cs | 1 + .../Agents/PlaywrightCliInstallerTests.cs | 27 ++++++++++--------- .../VsCodeAgentEnvironmentScannerTests.cs | 1 + 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs index 20995cfc1d0..6ad304b200c 100644 --- a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs +++ b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Security.Cryptography; +using Aspire.Cli.Interaction; using Aspire.Cli.Npm; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; @@ -16,6 +17,7 @@ internal sealed class PlaywrightCliInstaller( INpmRunner npmRunner, INpmProvenanceChecker provenanceChecker, IPlaywrightCliRunner playwrightCliRunner, + IInteractionService interactionService, IConfiguration configuration, ILogger logger) { @@ -63,6 +65,13 @@ internal sealed class PlaywrightCliInstaller( /// A token to cancel the operation. /// True if installation succeeded or was skipped (already up-to-date), false on failure. public async Task InstallAsync(CancellationToken cancellationToken) + { + return await interactionService.ShowStatusAsync( + "Installing Playwright CLI...", + () => InstallCoreAsync(cancellationToken)); + } + + private async Task InstallCoreAsync(CancellationToken cancellationToken) { // Step 1: Resolve the target version and integrity hash from the npm registry. var versionOverride = configuration[VersionOverrideKey]; diff --git a/tests/Aspire.Cli.Tests/Agents/ClaudeCodeAgentEnvironmentScannerTests.cs b/tests/Aspire.Cli.Tests/Agents/ClaudeCodeAgentEnvironmentScannerTests.cs index efbbe8f0ed5..a67a5428925 100644 --- a/tests/Aspire.Cli.Tests/Agents/ClaudeCodeAgentEnvironmentScannerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/ClaudeCodeAgentEnvironmentScannerTests.cs @@ -127,6 +127,7 @@ private static PlaywrightCliInstaller CreatePlaywrightCliInstaller() new FakeNpmRunner(), new FakeNpmProvenanceChecker(), new FakePlaywrightCliRunner(), + new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); } diff --git a/tests/Aspire.Cli.Tests/Agents/CopilotCliAgentEnvironmentScannerTests.cs b/tests/Aspire.Cli.Tests/Agents/CopilotCliAgentEnvironmentScannerTests.cs index 5b705e3c50c..398305f509b 100644 --- a/tests/Aspire.Cli.Tests/Agents/CopilotCliAgentEnvironmentScannerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/CopilotCliAgentEnvironmentScannerTests.cs @@ -329,6 +329,7 @@ private static PlaywrightCliInstaller CreatePlaywrightCliInstaller() new FakeNpmRunner(), new FakeNpmProvenanceChecker(), new FakePlaywrightCliRunner(), + new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); } diff --git a/tests/Aspire.Cli.Tests/Agents/OpenCodeAgentEnvironmentScannerTests.cs b/tests/Aspire.Cli.Tests/Agents/OpenCodeAgentEnvironmentScannerTests.cs index af64edf373b..6ec5bcf55e6 100644 --- a/tests/Aspire.Cli.Tests/Agents/OpenCodeAgentEnvironmentScannerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/OpenCodeAgentEnvironmentScannerTests.cs @@ -107,6 +107,7 @@ private static PlaywrightCliInstaller CreatePlaywrightCliInstaller() new FakeNpmRunner(), new FakeNpmProvenanceChecker(), new FakePlaywrightCliRunner(), + new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); } diff --git a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs index f4342d18ac9..8a1b2d8a041 100644 --- a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs @@ -4,6 +4,7 @@ using System.Security.Cryptography; using Aspire.Cli.Agents.Playwright; using Aspire.Cli.Npm; +using Aspire.Cli.Tests.TestServices; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; using Semver; @@ -20,7 +21,7 @@ public async Task InstallAsync_WhenNpmResolveReturnsNull_ReturnsFalse() ResolveResult = null }; var playwrightRunner = new TestPlaywrightCliRunner(); - var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -40,7 +41,7 @@ public async Task InstallAsync_WhenAlreadyInstalledAtSameVersion_SkipsInstallAnd InstalledVersion = version, InstallSkillsResult = true }; - var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -64,7 +65,7 @@ public async Task InstallAsync_WhenNewerVersionInstalled_SkipsInstallAndInstalls InstalledVersion = installedVersion, InstallSkillsResult = true }; - var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -83,7 +84,7 @@ public async Task InstallAsync_WhenPackFails_ReturnsFalse() PackResult = null }; var playwrightRunner = new TestPlaywrightCliRunner(); - var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -109,7 +110,7 @@ public async Task InstallAsync_WhenIntegrityCheckFails_ReturnsFalse() PackResult = tarballPath }; var playwrightRunner = new TestPlaywrightCliRunner(); - var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -149,7 +150,7 @@ public async Task InstallAsync_WhenIntegrityCheckPasses_InstallsGlobally() { InstallSkillsResult = true }; - var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -185,7 +186,7 @@ public async Task InstallAsync_WhenGlobalInstallFails_ReturnsFalse() InstallGlobalResult = false }; var playwrightRunner = new TestPlaywrightCliRunner(); - var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -225,7 +226,7 @@ public async Task InstallAsync_WhenOlderVersionInstalled_PerformsUpgrade() InstalledVersion = installedVersion, InstallSkillsResult = true }; - var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -303,7 +304,7 @@ public async Task InstallAsync_WhenAuditSignaturesFails_ReturnsFalse() }; var provenanceChecker = new TestNpmProvenanceChecker(); var playwrightRunner = new TestPlaywrightCliRunner(); - var installer = new PlaywrightCliInstaller(npmRunner, provenanceChecker, playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, provenanceChecker, playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -322,7 +323,7 @@ public async Task InstallAsync_WhenProvenanceCheckFails_ReturnsFalse() }; var provenanceChecker = new TestNpmProvenanceChecker { ProvenanceOutcome = ProvenanceVerificationOutcome.SourceRepositoryMismatch }; var playwrightRunner = new TestPlaywrightCliRunner(); - var installer = new PlaywrightCliInstaller(npmRunner, provenanceChecker, playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, provenanceChecker, playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -357,7 +358,7 @@ public async Task InstallAsync_WhenValidationDisabled_SkipsAllValidationChecks() [PlaywrightCliInstaller.DisablePackageValidationKey] = "true" }) .Build(); - var installer = new PlaywrightCliInstaller(npmRunner, provenanceChecker, playwrightRunner, configuration, NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, provenanceChecker, playwrightRunner, new TestConsoleInteractionService(), configuration, NullLogger.Instance); var result = await installer.InstallAsync(CancellationToken.None); @@ -387,7 +388,7 @@ public async Task InstallAsync_WhenVersionOverrideConfigured_UsesOverrideVersion [PlaywrightCliInstaller.VersionOverrideKey] = "0.2.0" }) .Build(); - var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, configuration, NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), configuration, NullLogger.Instance); await installer.InstallAsync(CancellationToken.None); @@ -403,7 +404,7 @@ public async Task InstallAsync_WhenNoVersionOverride_UsesDefaultRange() ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-abc123" } }; var playwrightRunner = new TestPlaywrightCliRunner(); - var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new ConfigurationBuilder().Build(), NullLogger.Instance); + var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); await installer.InstallAsync(CancellationToken.None); diff --git a/tests/Aspire.Cli.Tests/Agents/VsCodeAgentEnvironmentScannerTests.cs b/tests/Aspire.Cli.Tests/Agents/VsCodeAgentEnvironmentScannerTests.cs index 65c80be1ebe..41dea5f7079 100644 --- a/tests/Aspire.Cli.Tests/Agents/VsCodeAgentEnvironmentScannerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/VsCodeAgentEnvironmentScannerTests.cs @@ -367,6 +367,7 @@ private static PlaywrightCliInstaller CreatePlaywrightCliInstaller() new FakeNpmRunner(), new FakeNpmProvenanceChecker(), new FakePlaywrightCliRunner(), + new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); } From f1efa8dc5ef662d0d9ad5569c4fda5d88cbebe4d Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Fri, 27 Feb 2026 11:57:44 +1100 Subject: [PATCH 09/15] Mirror playwright-cli skill files to all detected agent environments After playwright-cli install --skills creates files in .claude/skills/, the installer now mirrors the playwright-cli skill directory to all other detected agent environment skill directories (.github/skills/, .opencode/skill/, etc.) so every configured environment has identical skill files. Stale files in target directories are also removed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Agents/AgentEnvironmentScanContext.cs | 16 +++ .../ClaudeCodeAgentEnvironmentScanner.cs | 5 +- .../Agents/CommonAgentApplicators.cs | 9 +- .../CopilotCliAgentEnvironmentScanner.cs | 5 +- .../OpenCodeAgentEnvironmentScanner.cs | 5 +- .../Playwright/PlaywrightCliInstaller.cs | 118 ++++++++++++++- .../VsCode/VsCodeAgentEnvironmentScanner.cs | 5 +- .../Agents/PlaywrightCliInstallerTests.cs | 136 ++++++++++++++++-- 8 files changed, 271 insertions(+), 28 deletions(-) diff --git a/src/Aspire.Cli/Agents/AgentEnvironmentScanContext.cs b/src/Aspire.Cli/Agents/AgentEnvironmentScanContext.cs index 8d43c58f7e1..3a5d22424d9 100644 --- a/src/Aspire.Cli/Agents/AgentEnvironmentScanContext.cs +++ b/src/Aspire.Cli/Agents/AgentEnvironmentScanContext.cs @@ -10,6 +10,7 @@ internal sealed class AgentEnvironmentScanContext { private readonly List _applicators = []; private readonly HashSet _skillFileApplicatorPaths = new(StringComparer.OrdinalIgnoreCase); + private readonly HashSet _skillBaseDirectories = new(StringComparer.OrdinalIgnoreCase); /// /// Gets the working directory being scanned. @@ -62,4 +63,19 @@ public void AddApplicator(AgentEnvironmentApplicator applicator) /// Gets the collection of detected applicators. /// public IReadOnlyList Applicators => _applicators; + + /// + /// Registers a skill base directory for an agent environment (e.g., ".claude/skills", ".github/skills"). + /// These directories are used to mirror skill files across all detected agent environments. + /// + /// The relative path to the skill base directory from the repository root. + public void AddSkillBaseDirectory(string relativeSkillBaseDir) + { + _skillBaseDirectories.Add(relativeSkillBaseDir); + } + + /// + /// Gets the registered skill base directories for all detected agent environments. + /// + public IReadOnlyCollection SkillBaseDirectories => _skillBaseDirectories; } diff --git a/src/Aspire.Cli/Agents/ClaudeCode/ClaudeCodeAgentEnvironmentScanner.cs b/src/Aspire.Cli/Agents/ClaudeCode/ClaudeCodeAgentEnvironmentScanner.cs index a4e225a0c2a..b58b181df32 100644 --- a/src/Aspire.Cli/Agents/ClaudeCode/ClaudeCodeAgentEnvironmentScanner.cs +++ b/src/Aspire.Cli/Agents/ClaudeCode/ClaudeCodeAgentEnvironmentScanner.cs @@ -18,6 +18,7 @@ internal sealed class ClaudeCodeAgentEnvironmentScanner : IAgentEnvironmentScann private const string McpConfigFileName = ".mcp.json"; private const string AspireServerName = "aspire"; private static readonly string s_skillFilePath = Path.Combine(".claude", "skills", CommonAgentApplicators.AspireSkillName, "SKILL.md"); + private static readonly string s_skillBaseDirectory = Path.Combine(".claude", "skills"); private const string SkillFileDescription = "Create Aspire skill file (.claude/skills/aspire/SKILL.md)"; private readonly IClaudeCodeCliRunner _claudeCodeCliRunner; @@ -74,7 +75,7 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok } // Register Playwright CLI installation applicator - CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller, s_skillBaseDirectory); // Try to add skill file applicator for Claude Code CommonAgentApplicators.TryAddSkillFileApplicator( @@ -105,7 +106,7 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok } // Register Playwright CLI installation applicator - CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller, s_skillBaseDirectory); // Try to add skill file applicator for Claude Code CommonAgentApplicators.TryAddSkillFileApplicator( diff --git a/src/Aspire.Cli/Agents/CommonAgentApplicators.cs b/src/Aspire.Cli/Agents/CommonAgentApplicators.cs index e2b8d070342..e47f82ee6e8 100644 --- a/src/Aspire.Cli/Agents/CommonAgentApplicators.cs +++ b/src/Aspire.Cli/Agents/CommonAgentApplicators.cs @@ -81,10 +81,15 @@ public static bool TryAddSkillFileApplicator( /// /// The scan context. /// The Playwright CLI installer that handles secure installation. + /// The relative path to the skill base directory for this agent environment (e.g., ".claude/skills", ".github/skills"). public static void AddPlaywrightCliApplicator( AgentEnvironmentScanContext context, - PlaywrightCliInstaller installer) + PlaywrightCliInstaller installer, + string skillBaseDirectory) { + // Register the skill base directory so skill files can be mirrored to all environments + context.AddSkillBaseDirectory(skillBaseDirectory); + // Only add the Playwright applicator prompt once across all environments if (context.PlaywrightApplicatorAdded) { @@ -94,7 +99,7 @@ public static void AddPlaywrightCliApplicator( context.PlaywrightApplicatorAdded = true; context.AddApplicator(new AgentEnvironmentApplicator( "Install Playwright CLI for browser automation", - installer.InstallAsync, + ct => installer.InstallAsync(context, ct), promptGroup: McpInitPromptGroup.AdditionalOptions, priority: 1)); } diff --git a/src/Aspire.Cli/Agents/CopilotCli/CopilotCliAgentEnvironmentScanner.cs b/src/Aspire.Cli/Agents/CopilotCli/CopilotCliAgentEnvironmentScanner.cs index 40d53114b4f..e5625598b11 100644 --- a/src/Aspire.Cli/Agents/CopilotCli/CopilotCliAgentEnvironmentScanner.cs +++ b/src/Aspire.Cli/Agents/CopilotCli/CopilotCliAgentEnvironmentScanner.cs @@ -18,6 +18,7 @@ internal sealed class CopilotCliAgentEnvironmentScanner : IAgentEnvironmentScann private const string McpConfigFileName = "mcp-config.json"; private const string AspireServerName = "aspire"; private static readonly string s_skillFilePath = Path.Combine(".github", "skills", CommonAgentApplicators.AspireSkillName, "SKILL.md"); + private static readonly string s_skillBaseDirectory = Path.Combine(".github", "skills"); private const string SkillFileDescription = "Create Aspire skill file (.github/skills/aspire/SKILL.md)"; private readonly ICopilotCliRunner _copilotCliRunner; @@ -73,7 +74,7 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok } // Register Playwright CLI installation applicator - CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller, s_skillBaseDirectory); // Try to add skill file applicator for GitHub Copilot CommonAgentApplicators.TryAddSkillFileApplicator( @@ -111,7 +112,7 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok } // Register Playwright CLI installation applicator - CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller, s_skillBaseDirectory); // Try to add skill file applicator for GitHub Copilot CommonAgentApplicators.TryAddSkillFileApplicator( diff --git a/src/Aspire.Cli/Agents/OpenCode/OpenCodeAgentEnvironmentScanner.cs b/src/Aspire.Cli/Agents/OpenCode/OpenCodeAgentEnvironmentScanner.cs index d2036b16470..98450ccae21 100644 --- a/src/Aspire.Cli/Agents/OpenCode/OpenCodeAgentEnvironmentScanner.cs +++ b/src/Aspire.Cli/Agents/OpenCode/OpenCodeAgentEnvironmentScanner.cs @@ -17,6 +17,7 @@ internal sealed class OpenCodeAgentEnvironmentScanner : IAgentEnvironmentScanner private const string OpenCodeConfigFileName = "opencode.jsonc"; private const string AspireServerName = "aspire"; private static readonly string s_skillFilePath = Path.Combine(".opencode", "skill", CommonAgentApplicators.AspireSkillName, "SKILL.md"); + private static readonly string s_skillBaseDirectory = Path.Combine(".opencode", "skill"); private const string SkillFileDescription = "Create Aspire skill file (.opencode/skill/aspire/SKILL.md)"; private readonly IOpenCodeCliRunner _openCodeCliRunner; @@ -68,7 +69,7 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok } // Register Playwright CLI installation applicator - CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller, s_skillBaseDirectory); // Try to add skill file applicator for OpenCode CommonAgentApplicators.TryAddSkillFileApplicator( @@ -91,7 +92,7 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok context.AddApplicator(CreateApplicator(configDirectory)); // Register Playwright CLI installation applicator - CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller, s_skillBaseDirectory); // Try to add skill file applicator for OpenCode CommonAgentApplicators.TryAddSkillFileApplicator( diff --git a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs index 6ad304b200c..af5ba40010c 100644 --- a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs +++ b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs @@ -47,6 +47,16 @@ internal sealed class PlaywrightCliInstaller( /// internal const string ExpectedBuildType = "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1"; + /// + /// The name of the playwright-cli skill directory. + /// + internal const string PlaywrightCliSkillName = "playwright-cli"; + + /// + /// The primary skill base directory where playwright-cli installs skills. + /// + internal static readonly string s_primarySkillBaseDirectory = Path.Combine(".claude", "skills"); + /// /// Configuration key that disables package validation when set to "true". /// This is a break-glass mechanism for debugging npm service issues and must never be the default. @@ -62,16 +72,17 @@ internal sealed class PlaywrightCliInstaller( /// /// Installs the Playwright CLI with supply chain verification and generates skill files. /// + /// The agent environment scan context containing detected skill directories. /// A token to cancel the operation. /// True if installation succeeded or was skipped (already up-to-date), false on failure. - public async Task InstallAsync(CancellationToken cancellationToken) + public async Task InstallAsync(AgentEnvironmentScanContext context, CancellationToken cancellationToken) { return await interactionService.ShowStatusAsync( "Installing Playwright CLI...", - () => InstallCoreAsync(cancellationToken)); + () => InstallCoreAsync(context, cancellationToken)); } - private async Task InstallCoreAsync(CancellationToken cancellationToken) + private async Task InstallCoreAsync(AgentEnvironmentScanContext context, CancellationToken cancellationToken) { // Step 1: Resolve the target version and integrity hash from the npm registry. var versionOverride = configuration[VersionOverrideKey]; @@ -106,7 +117,12 @@ private async Task InstallCoreAsync(CancellationToken cancellationToken) packageInfo.Version); // Still install skills in case they're missing. - return await playwrightCliRunner.InstallSkillsAsync(cancellationToken); + var skillsInstalled = await playwrightCliRunner.InstallSkillsAsync(cancellationToken); + if (skillsInstalled) + { + MirrorSkillFiles(context); + } + return skillsInstalled; } logger.LogDebug( @@ -213,7 +229,12 @@ private async Task InstallCoreAsync(CancellationToken cancellationToken) // Step 8: Generate skill files. logger.LogDebug("Generating Playwright CLI skill files"); - return await playwrightCliRunner.InstallSkillsAsync(cancellationToken); + var skillsResult = await playwrightCliRunner.InstallSkillsAsync(cancellationToken); + if (skillsResult) + { + MirrorSkillFiles(context); + } + return skillsResult; } finally { @@ -232,6 +253,93 @@ private async Task InstallCoreAsync(CancellationToken cancellationToken) } } + /// + /// Mirrors the playwright-cli skill directory from the primary location to all other + /// detected agent environment skill directories so that every configured environment + /// has an identical copy of the skill files. + /// + private void MirrorSkillFiles(AgentEnvironmentScanContext context) + { + var repoRoot = context.RepositoryRoot.FullName; + var primarySkillDir = Path.Combine(repoRoot, s_primarySkillBaseDirectory, PlaywrightCliSkillName); + + if (!Directory.Exists(primarySkillDir)) + { + logger.LogDebug("Primary skill directory does not exist: {PrimarySkillDir}", primarySkillDir); + return; + } + + foreach (var skillBaseDir in context.SkillBaseDirectories) + { + // Skip the primary directory — it's the source + if (string.Equals(skillBaseDir, s_primarySkillBaseDirectory, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var targetSkillDir = Path.Combine(repoRoot, skillBaseDir, PlaywrightCliSkillName); + + try + { + SyncDirectory(primarySkillDir, targetSkillDir); + logger.LogDebug("Mirrored playwright-cli skills to {TargetDir}", targetSkillDir); + } + catch (IOException ex) + { + logger.LogWarning(ex, "Failed to mirror playwright-cli skills to {TargetDir}", targetSkillDir); + } + } + } + + /// + /// Synchronizes the contents of the source directory to the target directory, + /// creating, updating, and removing files so the target matches the source exactly. + /// + internal static void SyncDirectory(string sourceDir, string targetDir) + { + Directory.CreateDirectory(targetDir); + + // Copy all files from source to target + foreach (var sourceFile in Directory.GetFiles(sourceDir, "*", SearchOption.AllDirectories)) + { + var relativePath = Path.GetRelativePath(sourceDir, sourceFile); + var targetFile = Path.Combine(targetDir, relativePath); + + var targetFileDir = Path.GetDirectoryName(targetFile); + if (!string.IsNullOrEmpty(targetFileDir)) + { + Directory.CreateDirectory(targetFileDir); + } + + File.Copy(sourceFile, targetFile, overwrite: true); + } + + // Remove files in target that don't exist in source + if (Directory.Exists(targetDir)) + { + foreach (var targetFile in Directory.GetFiles(targetDir, "*", SearchOption.AllDirectories)) + { + var relativePath = Path.GetRelativePath(targetDir, targetFile); + var sourceFile = Path.Combine(sourceDir, relativePath); + + if (!File.Exists(sourceFile)) + { + File.Delete(targetFile); + } + } + + // Remove empty directories in target + foreach (var dir in Directory.GetDirectories(targetDir, "*", SearchOption.AllDirectories) + .OrderByDescending(d => d.Length)) + { + if (Directory.Exists(dir) && Directory.GetFileSystemEntries(dir).Length == 0) + { + Directory.Delete(dir); + } + } + } + } + /// /// Verifies that the SHA-512 hash of the file matches the SRI integrity string. /// diff --git a/src/Aspire.Cli/Agents/VsCode/VsCodeAgentEnvironmentScanner.cs b/src/Aspire.Cli/Agents/VsCode/VsCodeAgentEnvironmentScanner.cs index 22c8774127d..27d24c1dd7e 100644 --- a/src/Aspire.Cli/Agents/VsCode/VsCodeAgentEnvironmentScanner.cs +++ b/src/Aspire.Cli/Agents/VsCode/VsCodeAgentEnvironmentScanner.cs @@ -18,6 +18,7 @@ internal sealed class VsCodeAgentEnvironmentScanner : IAgentEnvironmentScanner private const string McpConfigFileName = "mcp.json"; private const string AspireServerName = "aspire"; private static readonly string s_skillFilePath = Path.Combine(".github", "skills", CommonAgentApplicators.AspireSkillName, "SKILL.md"); + private static readonly string s_skillBaseDirectory = Path.Combine(".github", "skills"); private const string SkillFileDescription = "Create Aspire skill file (.github/skills/aspire/SKILL.md)"; private readonly IVsCodeCliRunner _vsCodeCliRunner; @@ -70,7 +71,7 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok } // Register Playwright CLI installation applicator - CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller, s_skillBaseDirectory); // Try to add skill file applicator for GitHub Copilot CommonAgentApplicators.TryAddSkillFileApplicator( @@ -89,7 +90,7 @@ public async Task ScanAsync(AgentEnvironmentScanContext context, CancellationTok context.AddApplicator(CreateAspireApplicator(targetVsCodeFolder)); // Register Playwright CLI installation applicator - CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller); + CommonAgentApplicators.AddPlaywrightCliApplicator(context, _playwrightCliInstaller, s_skillBaseDirectory); // Try to add skill file applicator for GitHub Copilot CommonAgentApplicators.TryAddSkillFileApplicator( diff --git a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs index 8a1b2d8a041..7c92fc3cdc9 100644 --- a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Security.Cryptography; +using Aspire.Cli.Agents; using Aspire.Cli.Agents.Playwright; using Aspire.Cli.Npm; using Aspire.Cli.Tests.TestServices; @@ -13,6 +14,17 @@ namespace Aspire.Cli.Tests.Agents; public class PlaywrightCliInstallerTests { + private static AgentEnvironmentScanContext CreateTestContext() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"aspire-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + return new AgentEnvironmentScanContext + { + WorkingDirectory = new DirectoryInfo(tempDir), + RepositoryRoot = new DirectoryInfo(tempDir) + }; + } + [Fact] public async Task InstallAsync_WhenNpmResolveReturnsNull_ReturnsFalse() { @@ -23,7 +35,7 @@ public async Task InstallAsync_WhenNpmResolveReturnsNull_ReturnsFalse() var playwrightRunner = new TestPlaywrightCliRunner(); var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); - var result = await installer.InstallAsync(CancellationToken.None); + var result = await installer.InstallAsync(CreateTestContext(), CancellationToken.None); Assert.False(result); } @@ -43,7 +55,7 @@ public async Task InstallAsync_WhenAlreadyInstalledAtSameVersion_SkipsInstallAnd }; var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); - var result = await installer.InstallAsync(CancellationToken.None); + var result = await installer.InstallAsync(CreateTestContext(), CancellationToken.None); Assert.True(result); Assert.True(playwrightRunner.InstallSkillsCalled); @@ -67,7 +79,7 @@ public async Task InstallAsync_WhenNewerVersionInstalled_SkipsInstallAndInstalls }; var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); - var result = await installer.InstallAsync(CancellationToken.None); + var result = await installer.InstallAsync(CreateTestContext(), CancellationToken.None); Assert.True(result); Assert.True(playwrightRunner.InstallSkillsCalled); @@ -86,7 +98,7 @@ public async Task InstallAsync_WhenPackFails_ReturnsFalse() var playwrightRunner = new TestPlaywrightCliRunner(); var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); - var result = await installer.InstallAsync(CancellationToken.None); + var result = await installer.InstallAsync(CreateTestContext(), CancellationToken.None); Assert.False(result); Assert.True(npmRunner.PackCalled); @@ -112,7 +124,7 @@ public async Task InstallAsync_WhenIntegrityCheckFails_ReturnsFalse() var playwrightRunner = new TestPlaywrightCliRunner(); var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); - var result = await installer.InstallAsync(CancellationToken.None); + var result = await installer.InstallAsync(CreateTestContext(), CancellationToken.None); Assert.False(result); Assert.False(npmRunner.InstallGlobalCalled); @@ -152,7 +164,7 @@ public async Task InstallAsync_WhenIntegrityCheckPasses_InstallsGlobally() }; var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); - var result = await installer.InstallAsync(CancellationToken.None); + var result = await installer.InstallAsync(CreateTestContext(), CancellationToken.None); Assert.True(result); Assert.True(npmRunner.InstallGlobalCalled); @@ -188,7 +200,7 @@ public async Task InstallAsync_WhenGlobalInstallFails_ReturnsFalse() var playwrightRunner = new TestPlaywrightCliRunner(); var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); - var result = await installer.InstallAsync(CancellationToken.None); + var result = await installer.InstallAsync(CreateTestContext(), CancellationToken.None); Assert.False(result); } @@ -228,7 +240,7 @@ public async Task InstallAsync_WhenOlderVersionInstalled_PerformsUpgrade() }; var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); - var result = await installer.InstallAsync(CancellationToken.None); + var result = await installer.InstallAsync(CreateTestContext(), CancellationToken.None); Assert.True(result); Assert.True(npmRunner.PackCalled); @@ -306,7 +318,7 @@ public async Task InstallAsync_WhenAuditSignaturesFails_ReturnsFalse() var playwrightRunner = new TestPlaywrightCliRunner(); var installer = new PlaywrightCliInstaller(npmRunner, provenanceChecker, playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); - var result = await installer.InstallAsync(CancellationToken.None); + var result = await installer.InstallAsync(CreateTestContext(), CancellationToken.None); Assert.False(result); Assert.False(provenanceChecker.ProvenanceCalled); @@ -325,7 +337,7 @@ public async Task InstallAsync_WhenProvenanceCheckFails_ReturnsFalse() var playwrightRunner = new TestPlaywrightCliRunner(); var installer = new PlaywrightCliInstaller(npmRunner, provenanceChecker, playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); - var result = await installer.InstallAsync(CancellationToken.None); + var result = await installer.InstallAsync(CreateTestContext(), CancellationToken.None); Assert.False(result); Assert.True(provenanceChecker.ProvenanceCalled); @@ -360,7 +372,7 @@ public async Task InstallAsync_WhenValidationDisabled_SkipsAllValidationChecks() .Build(); var installer = new PlaywrightCliInstaller(npmRunner, provenanceChecker, playwrightRunner, new TestConsoleInteractionService(), configuration, NullLogger.Instance); - var result = await installer.InstallAsync(CancellationToken.None); + var result = await installer.InstallAsync(CreateTestContext(), CancellationToken.None); Assert.True(result); Assert.False(provenanceChecker.ProvenanceCalled); @@ -390,7 +402,7 @@ public async Task InstallAsync_WhenVersionOverrideConfigured_UsesOverrideVersion .Build(); var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), configuration, NullLogger.Instance); - await installer.InstallAsync(CancellationToken.None); + await installer.InstallAsync(CreateTestContext(), CancellationToken.None); Assert.Equal("0.2.0", npmRunner.ResolvedVersionRange); } @@ -406,11 +418,109 @@ public async Task InstallAsync_WhenNoVersionOverride_UsesDefaultRange() var playwrightRunner = new TestPlaywrightCliRunner(); var installer = new PlaywrightCliInstaller(npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); - await installer.InstallAsync(CancellationToken.None); + await installer.InstallAsync(CreateTestContext(), CancellationToken.None); Assert.Equal(PlaywrightCliInstaller.VersionRange, npmRunner.ResolvedVersionRange); } + [Fact] + public async Task InstallAsync_MirrorsSkillFilesToOtherAgentEnvironments() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"aspire-mirror-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + + try + { + // Set up the primary skill directory with a skill file (simulating playwright-cli output) + var primarySkillDir = Path.Combine(tempDir, ".claude", "skills", "playwright-cli"); + Directory.CreateDirectory(primarySkillDir); + await File.WriteAllTextAsync(Path.Combine(primarySkillDir, "SKILL.md"), "# Playwright CLI Skill"); + Directory.CreateDirectory(Path.Combine(primarySkillDir, "subdir")); + await File.WriteAllTextAsync(Path.Combine(primarySkillDir, "subdir", "extra.md"), "Extra content"); + + var version = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); + var playwrightRunner = new TestPlaywrightCliRunner + { + InstalledVersion = version, + InstallSkillsResult = true + }; + var npmRunner = new TestNpmRunner + { + ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-abc123" } + }; + + var installer = new PlaywrightCliInstaller( + npmRunner, new TestNpmProvenanceChecker(), playwrightRunner, + new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), + NullLogger.Instance); + + var context = new AgentEnvironmentScanContext + { + WorkingDirectory = new DirectoryInfo(tempDir), + RepositoryRoot = new DirectoryInfo(tempDir) + }; + context.AddSkillBaseDirectory(Path.Combine(".claude", "skills")); + context.AddSkillBaseDirectory(Path.Combine(".github", "skills")); + context.AddSkillBaseDirectory(Path.Combine(".opencode", "skill")); + + await installer.InstallAsync(context, CancellationToken.None); + + // Verify files were mirrored to .github/skills/playwright-cli/ + Assert.True(File.Exists(Path.Combine(tempDir, ".github", "skills", "playwright-cli", "SKILL.md"))); + Assert.True(File.Exists(Path.Combine(tempDir, ".github", "skills", "playwright-cli", "subdir", "extra.md"))); + Assert.Equal("# Playwright CLI Skill", await File.ReadAllTextAsync(Path.Combine(tempDir, ".github", "skills", "playwright-cli", "SKILL.md"))); + + // Verify files were mirrored to .opencode/skill/playwright-cli/ + Assert.True(File.Exists(Path.Combine(tempDir, ".opencode", "skill", "playwright-cli", "SKILL.md"))); + Assert.True(File.Exists(Path.Combine(tempDir, ".opencode", "skill", "playwright-cli", "subdir", "extra.md"))); + } + finally + { + if (Directory.Exists(tempDir)) + { + Directory.Delete(tempDir, recursive: true); + } + } + } + + [Fact] + public void SyncDirectory_RemovesExtraFilesInTarget() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"aspire-sync-test-{Guid.NewGuid():N}"); + var sourceDir = Path.Combine(tempDir, "source"); + var targetDir = Path.Combine(tempDir, "target"); + + try + { + // Set up source with one file + Directory.CreateDirectory(sourceDir); + File.WriteAllText(Path.Combine(sourceDir, "keep.md"), "keep"); + + // Set up target with an extra file that should be removed + Directory.CreateDirectory(targetDir); + File.WriteAllText(Path.Combine(targetDir, "keep.md"), "old content"); + File.WriteAllText(Path.Combine(targetDir, "stale.md"), "should be removed"); + Directory.CreateDirectory(Path.Combine(targetDir, "stale-dir")); + File.WriteAllText(Path.Combine(targetDir, "stale-dir", "old.md"), "should be removed"); + + PlaywrightCliInstaller.SyncDirectory(sourceDir, targetDir); + + // Source file should be copied + Assert.Equal("keep", File.ReadAllText(Path.Combine(targetDir, "keep.md"))); + + // Stale files and directories should be removed + Assert.False(File.Exists(Path.Combine(targetDir, "stale.md"))); + Assert.False(Directory.Exists(Path.Combine(targetDir, "stale-dir"))); + } + finally + { + if (Directory.Exists(tempDir)) + { + Directory.Delete(tempDir, recursive: true); + } + } + } + private sealed class TestNpmRunner : INpmRunner { public NpmPackageInfo? ResolveResult { get; set; } From 54c40244d73f5412279d7b778cd3c74a4272073b Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Fri, 27 Feb 2026 13:22:55 +1100 Subject: [PATCH 10/15] Update security design doc to match implementation - Step 4 now documents all three provenance gates: source repository, workflow path, and build type (with OIDC issuer implication) - Added table of verified fields with expected values - Updated implementation constants to include new fields - Added configuration section documenting break-glass keys - Updated verification diagram with workflow/build type checks - Step 7 now documents skill file mirroring across environments - Future improvements reflects experimental Sigstore branch status Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/specs/safe-npm-tool-install.md | 73 +++++++++++++++++++---------- 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/docs/specs/safe-npm-tool-install.md b/docs/specs/safe-npm-tool-install.md index 16893f933e6..56b29cf4c4c 100644 --- a/docs/specs/safe-npm-tool-install.md +++ b/docs/specs/safe-npm-tool-install.md @@ -35,7 +35,7 @@ Our verification chain relies on these trust anchors: ### Step 1: Resolve package version and metadata -**Action:** Run `npm view @playwright/cli@{versionRange} version` and `npm view @playwright/cli@{version} dist.integrity` to get the resolved version and the registry's SRI integrity hash. +**Action:** Run `npm view @playwright/cli@{versionRange} version` and `npm view @playwright/cli@{version} dist.integrity` to get the resolved version and the registry's SRI integrity hash. The default version range is `>=0.1.1`, which resolves to the latest published version at or above 0.1.1. This can be overridden to a specific version via the `playwrightCliVersion` configuration key. **What this establishes:** We know the exact version we intend to install and the hash the registry claims for its tarball. @@ -71,18 +71,27 @@ Our verification chain relies on these trust anchors: **Limitations:** `npm audit signatures` verifies that *valid attestations exist* but does not expose the attestation *content*. It confirms "this package has authentic Sigstore-signed attestations" but does not tell us *what* those attestations say (e.g., which repository built the package). That's addressed in Step 4. -### Step 4: Verify provenance source repository +### Step 4: Verify provenance metadata **Action:** 1. Fetch the attestation bundle from `https://registry.npmjs.org/-/npm/v1/attestations/@playwright/cli@{version}` 2. Find the attestation with `predicateType: "https://slsa.dev/provenance/v1"` (SLSA Build L3 provenance) 3. Base64-decode the DSSE envelope payload to extract the in-toto statement -4. Parse `predicate.buildDefinition.externalParameters.workflow.repository` -5. Verify it equals `https://github.com/microsoft/playwright-cli` +4. Verify the following fields from the provenance predicate: -**What this establishes:** That the Sigstore-attested provenance for this package version claims it was built from the `microsoft/playwright-cli` GitHub repository via a GitHub Actions workflow. +| Field | Location in payload | Expected value | What it proves | +|---|---|---|---| +| **Source repository** | `predicate.buildDefinition.externalParameters.workflow.repository` | `https://github.com/microsoft/playwright-cli` | The package was built from the legitimate source code | +| **Workflow path** | `predicate.buildDefinition.externalParameters.workflow.path` | `.github/workflows/publish.yml` | The build used the expected CI pipeline, not an ad-hoc or attacker-injected workflow | +| **Build type** | `predicate.buildDefinition.buildType` | `https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1` | The build ran on GitHub Actions, which implicitly confirms the OIDC token issuer is `https://token.actions.githubusercontent.com` | -**Trust basis:** The attestation content is protected by the Sigstore signature verified in Step 3. Since Step 3 confirmed the attestation is cryptographically authentic (signed by a valid Sigstore certificate corresponding to a GitHub Actions OIDC identity), the content we read in Step 4 cannot have been tampered with by the npm registry. An attacker would need to compromise Sigstore itself to forge attestation content pointing to `microsoft/playwright-cli`. +**What this establishes:** That the Sigstore-attested provenance for this package version claims it was built from the `microsoft/playwright-cli` GitHub repository, using the expected publish workflow, on the GitHub Actions CI system. The build type verification implicitly confirms the OIDC token issuer without needing to parse the Sigstore certificate directly — the SLSA build type `https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1` is only valid for GitHub Actions builds that obtain their signing credentials via GitHub's OIDC provider (`https://token.actions.githubusercontent.com`). + +**Additional fields extracted but not directly verified:** The provenance parser also extracts `workflow.ref` (e.g., `refs/tags/v0.1.1`) and `runDetails.builder.id` from the attestation. These are available in the `NpmProvenanceData` result for logging and diagnostics but are not currently used as verification gates. + +**Trust basis:** The attestation content is protected by the Sigstore signature verified in Step 3. Since Step 3 confirmed the attestation is cryptographically authentic (signed by a valid Sigstore certificate corresponding to a GitHub Actions OIDC identity), the content we read in Step 4 cannot have been tampered with by the npm registry. An attacker would need to compromise Sigstore itself to forge attestation content pointing to `microsoft/playwright-cli` with the correct workflow and build type. + +**Why we verify all three fields:** Checking only the source repository would leave a gap where an attacker with write access to the repo could introduce a malicious workflow (e.g., `.github/workflows/evil.yml`) that builds a tampered package. By also verifying the workflow path and build type, we ensure the package was built by the specific, expected CI pipeline running on the expected CI system. **Why we fetch from the registry API:** The npm CLI (`npm audit signatures`) verifies Sigstore signatures but does not expose provenance content in its output. The `--json` flag only produces `{"invalid":[],"missing":[]}`. There is no npm CLI command to read the source repository from a SLSA provenance attestation. We must fetch the attestation bundle directly from the registry API. @@ -109,20 +118,25 @@ Our verification chain relies on these trust anchors: **Trust basis:** All preceding verification steps have passed. The tarball content has been verified against the registry's published hash (Step 5), the Sigstore attestations for that content are cryptographically valid (Step 3), and the attestations claim the correct source repository (Step 4). -### Step 7: Generate skill files +### Step 7: Generate and mirror skill files -**Action:** Run `playwright-cli install --skills` to generate agent skill files. +**Action:** Run `playwright-cli install --skills` to generate agent skill files in the primary skill directory (`.claude/skills/playwright-cli/`), then mirror the skill directory to all other detected agent environment skill directories (e.g., `.github/skills/playwright-cli/`, `.opencode/skill/playwright-cli/`). The mirror is a full sync — files are created, updated, and stale files are removed so all environments have identical skill content. -**What this establishes:** The Playwright CLI skill files are available for agent environments. +**What this establishes:** The Playwright CLI skill files are available for all configured agent environments. ## Verification Chain Summary ```text - │ Hardcoded expectations │ - │ • Package: @playwright/cli │ - │ • Version range: 0.1.1 │ - │ • Source: microsoft/ │ - │ playwright-cli │ + ┌──────────────────────────────┐ + │ Hardcoded expectations │ + │ • Package: @playwright/cli │ + │ • Version range: >=0.1.1 │ + │ • Source: microsoft/ │ + │ playwright-cli │ + │ • Workflow: .github/ │ + │ workflows/publish.yml │ + │ • Build type: GitHub Actions │ + │ workflow/v1 │ └──────────────┬────────────────┘ │ ┌──────────────▼────────────────┐ @@ -134,13 +148,14 @@ Our verification chain relies on these trust anchors: │ │ │ ┌──────────▼──────────┐ ┌─────▼──────────┐ ┌──────▼──────────┐ │ Step 3: npm audit │ │ Step 4: Verify │ │ Step 5: npm pack│ - │ signatures │ │ provenance repo │ │ + SHA-512 check │ - │ (Sigstore crypto) │ │ (attestation │ │ (tarball │ - │ │ │ content) │ │ integrity) │ + │ signatures │ │ provenance │ │ + SHA-512 check │ + │ (Sigstore crypto) │ │ (repo, workflow │ │ (tarball │ + │ │ │ + build type) │ │ integrity) │ └──────────┬───────────┘ └─────┬──────────┘ └──────┬──────────┘ │ │ │ │ Attestation is │ Built from │ Tarball matches - │ authentic │ expected repo │ published hash + │ authentic │ expected repo + │ published hash + │ │ expected pipeline │ └────────────────────┼─────────────────────┘ │ ┌──────────────▼────────────────┐ @@ -175,15 +190,25 @@ Our verification chain relies on these trust anchors: ```csharp internal const string PackageName = "@playwright/cli"; -internal const string VersionRange = "0.1.1"; +internal const string VersionRange = ">=0.1.1"; internal const string ExpectedSourceRepository = "https://github.com/microsoft/playwright-cli"; -internal const string NpmRegistryAttestationsUrl = "https://registry.npmjs.org/-/npm/v1/attestations"; +internal const string ExpectedWorkflowPath = ".github/workflows/publish.yml"; +internal const string ExpectedBuildType = "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1"; +internal const string NpmRegistryAttestationsBaseUrl = "https://registry.npmjs.org/-/npm/v1/attestations"; internal const string SlsaProvenancePredicateType = "https://slsa.dev/provenance/v1"; ``` +## Configuration + +Two break-glass configuration keys are available via `aspire config set`: + +| Key | Effect | +|---|---| +| `disablePlaywrightCliPackageValidation` | When `"true"`, skips all Sigstore, provenance, and integrity checks. Use only for debugging npm service issues. | +| `playwrightCliVersion` | When set, overrides the version range and pins to the specified exact version. | + ## Future Improvements -1. **Direct Sigstore verification in C#** — Eliminate the dependency on `npm audit signatures` by implementing Sigstore bundle verification natively. This would remove residual risk #1 and reduce the dependency on npm CLI. -2. **Rekor log verification** — Independently verify the Rekor inclusion proof to confirm the attestation was logged in the public transparency log. -3. **Certificate identity verification** — Check the OIDC identity claims in the Sigstore signing certificate (issuer, subject, workflow reference) rather than just the provenance payload. -4. **Pinned tarball hash** — Ship a known-good SRI hash with each Aspire release, eliminating the need to trust the registry for the hash at all. +1. **Direct Sigstore verification in .NET** — An experimental implementation exists on the `sigstore-builtin-verification` branch using the [`Sigstore`](https://github.com/mitchdenny/sigstore-dotnet) .NET library. This eliminates residual risk #1 by performing Sigstore bundle verification natively (Fulcio certificate chain, Rekor transparency log inclusion proof, SCT verification, DSSE signature verification) without relying on `npm audit signatures`. It is gated behind the `builtInSigstoreVerificationEnabled` feature flag. +2. **Certificate identity verification** — The experimental Sigstore path already verifies the OIDC identity claims in the Sigstore signing certificate (issuer and SAN) via `CertificateIdentity.ForGitHubActions(repository)`. Bringing this to the default path would provide defense-in-depth beyond the provenance payload checks. +3. **Pinned tarball hash** — Ship a known-good SRI hash with each Aspire release, eliminating the need to trust the registry for the hash at all. From 2deb6de2a9a957a7b9a6db335ad90f02f86bac6b Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Fri, 27 Feb 2026 13:26:32 +1100 Subject: [PATCH 11/15] Verify workflow ref matches package version in provenance Add Gate 6 to provenance verification: check that the workflow ref (git tag) in the SLSA attestation matches refs/tags/v{version}. This ensures the build was triggered from the expected release tag, not an arbitrary branch or commit. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/specs/safe-npm-tool-install.md | 5 +- src/Aspire.Cli/Npm/INpmProvenanceChecker.cs | 8 ++- src/Aspire.Cli/Npm/NpmProvenanceChecker.cs | 16 +++++ .../Agents/NpmProvenanceCheckerTests.cs | 62 ++++++++++++++++++- 4 files changed, 86 insertions(+), 5 deletions(-) diff --git a/docs/specs/safe-npm-tool-install.md b/docs/specs/safe-npm-tool-install.md index 56b29cf4c4c..0beb331bcbb 100644 --- a/docs/specs/safe-npm-tool-install.md +++ b/docs/specs/safe-npm-tool-install.md @@ -84,10 +84,11 @@ Our verification chain relies on these trust anchors: | **Source repository** | `predicate.buildDefinition.externalParameters.workflow.repository` | `https://github.com/microsoft/playwright-cli` | The package was built from the legitimate source code | | **Workflow path** | `predicate.buildDefinition.externalParameters.workflow.path` | `.github/workflows/publish.yml` | The build used the expected CI pipeline, not an ad-hoc or attacker-injected workflow | | **Build type** | `predicate.buildDefinition.buildType` | `https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1` | The build ran on GitHub Actions, which implicitly confirms the OIDC token issuer is `https://token.actions.githubusercontent.com` | +| **Workflow ref** | `predicate.buildDefinition.externalParameters.workflow.ref` | `refs/tags/v{version}` (e.g., `refs/tags/v0.1.1`) | The build was triggered from a version tag matching the package version, not an arbitrary branch or commit | -**What this establishes:** That the Sigstore-attested provenance for this package version claims it was built from the `microsoft/playwright-cli` GitHub repository, using the expected publish workflow, on the GitHub Actions CI system. The build type verification implicitly confirms the OIDC token issuer without needing to parse the Sigstore certificate directly — the SLSA build type `https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1` is only valid for GitHub Actions builds that obtain their signing credentials via GitHub's OIDC provider (`https://token.actions.githubusercontent.com`). +**What this establishes:** That the Sigstore-attested provenance for this package version claims it was built from the `microsoft/playwright-cli` GitHub repository, using the expected publish workflow, on the GitHub Actions CI system, triggered by a git tag that matches the package version. The build type verification implicitly confirms the OIDC token issuer without needing to parse the Sigstore certificate directly — the SLSA build type `https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1` is only valid for GitHub Actions builds that obtain their signing credentials via GitHub's OIDC provider (`https://token.actions.githubusercontent.com`). The workflow ref verification ensures the build corresponds to an explicit version release, preventing an attacker from publishing a package built from an arbitrary branch or commit. -**Additional fields extracted but not directly verified:** The provenance parser also extracts `workflow.ref` (e.g., `refs/tags/v0.1.1`) and `runDetails.builder.id` from the attestation. These are available in the `NpmProvenanceData` result for logging and diagnostics but are not currently used as verification gates. +**Additional fields extracted but not directly verified:** The provenance parser also extracts `runDetails.builder.id` from the attestation. This is available in the `NpmProvenanceData` result for logging and diagnostics but is not currently used as a verification gate. **Trust basis:** The attestation content is protected by the Sigstore signature verified in Step 3. Since Step 3 confirmed the attestation is cryptographically authentic (signed by a valid Sigstore certificate corresponding to a GitHub Actions OIDC identity), the content we read in Step 4 cannot have been tampered with by the npm registry. An attacker would need to compromise Sigstore itself to forge attestation content pointing to `microsoft/playwright-cli` with the correct workflow and build type. diff --git a/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs b/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs index fc0635633fa..151c33145b3 100644 --- a/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs +++ b/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs @@ -53,7 +53,13 @@ internal enum ProvenanceVerificationOutcome /// The SLSA build type does not match the expected GitHub Actions build type, /// indicating the package was not built by the expected CI system. /// - BuildTypeMismatch + BuildTypeMismatch, + + /// + /// The workflow ref (git tag) does not match the expected version tag (e.g., refs/tags/v{version}), + /// indicating the build was not triggered from the expected release tag. + /// + WorkflowRefMismatch } /// diff --git a/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs b/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs index 3aca8091f21..a95e50335b6 100644 --- a/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs +++ b/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs @@ -115,6 +115,22 @@ public async Task VerifyProvenanceAsync(string pac }; } + // Gate 6: Verify the workflow ref corresponds to a version tag matching the package version. + var expectedRef = $"refs/tags/v{version}"; + if (!string.Equals(provenance.WorkflowRef, expectedRef, StringComparison.Ordinal)) + { + logger.LogWarning( + "Provenance verification failed: expected workflow ref {Expected} but attestation says {Actual}", + expectedRef, + provenance.WorkflowRef); + + return new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.WorkflowRefMismatch, + Provenance = provenance + }; + } + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.Verified, diff --git a/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs b/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs index 130ff0e0770..2e8b3ce36ba 100644 --- a/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs @@ -23,6 +23,7 @@ public void ParseProvenance_WithValidSlsaProvenance_ReturnsVerifiedWithData() Assert.Equal(".github/workflows/publish.yml", result.Value.Provenance.WorkflowPath); Assert.Equal("https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", result.Value.Provenance.BuildType); Assert.Equal("https://github.com/actions/runner/github-hosted", result.Value.Provenance.BuilderId); + Assert.Equal("refs/tags/v0.1.1", result.Value.Provenance.WorkflowRef); } [Fact] @@ -138,7 +139,52 @@ public void ParseProvenance_WithMissingPayload_ReturnsPayloadDecodeFailed() Assert.Equal(ProvenanceVerificationOutcome.PayloadDecodeFailed, result.Value.Outcome); } - private static string BuildAttestationJson(string sourceRepository, string workflowPath = ".github/workflows/publish.yml", string buildType = "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1") + [Fact] + public async Task VerifyProvenanceAsync_WithMismatchedWorkflowRef_ReturnsWorkflowRefMismatch() + { + var json = BuildAttestationJson( + "https://github.com/microsoft/playwright-cli", + workflowRef: "refs/tags/v9.9.9"); + + var handler = new TestHttpMessageHandler(json); + var httpClient = new HttpClient(handler); + var checker = new NpmProvenanceChecker(httpClient, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + var result = await checker.VerifyProvenanceAsync( + "@playwright/cli", + "0.1.1", + "https://github.com/microsoft/playwright-cli", + ".github/workflows/publish.yml", + "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + CancellationToken.None); + + Assert.Equal(ProvenanceVerificationOutcome.WorkflowRefMismatch, result.Outcome); + Assert.Equal("refs/tags/v9.9.9", result.Provenance?.WorkflowRef); + } + + [Fact] + public async Task VerifyProvenanceAsync_WithMatchingWorkflowRef_ReturnsVerified() + { + var json = BuildAttestationJson( + "https://github.com/microsoft/playwright-cli", + workflowRef: "refs/tags/v0.1.1"); + + var handler = new TestHttpMessageHandler(json); + var httpClient = new HttpClient(handler); + var checker = new NpmProvenanceChecker(httpClient, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + var result = await checker.VerifyProvenanceAsync( + "@playwright/cli", + "0.1.1", + "https://github.com/microsoft/playwright-cli", + ".github/workflows/publish.yml", + "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + CancellationToken.None); + + Assert.Equal(ProvenanceVerificationOutcome.Verified, result.Outcome); + } + + private static string BuildAttestationJson(string sourceRepository, string workflowPath = ".github/workflows/publish.yml", string buildType = "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", string workflowRef = "refs/tags/v0.1.1") { var statement = new JsonObject { @@ -154,7 +200,8 @@ private static string BuildAttestationJson(string sourceRepository, string workf ["workflow"] = new JsonObject { ["repository"] = sourceRepository, - ["path"] = workflowPath + ["path"] = workflowPath, + ["ref"] = workflowRef } } }, @@ -190,4 +237,15 @@ private static string BuildAttestationJson(string sourceRepository, string workf return attestationResponse.ToJsonString(); } + + private sealed class TestHttpMessageHandler(string responseContent) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(responseContent, Encoding.UTF8, "application/json") + }); + } + } } From 8ffef670c7575f41d72e5a972fdc22906b86d409 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Fri, 27 Feb 2026 15:34:12 +1100 Subject: [PATCH 12/15] Use callback-based workflow ref validation in provenance checker Make the workflow ref validation configurable per-package by accepting a Func callback instead of hardcoding the refs/tags/v{version} format. The ref is parsed into a WorkflowRefInfo record (Raw, Kind, Name) and the caller decides what's valid. PlaywrightCliInstaller validates Kind=tags and Name=v{version}. Other packages can use different tag conventions without modifying the provenance checker. Addresses review feedback from DamianEdwards. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/specs/safe-npm-tool-install.md | 2 +- .../Playwright/PlaywrightCliInstaller.cs | 2 + src/Aspire.Cli/Npm/INpmProvenanceChecker.cs | 56 +++++++++++++++++- src/Aspire.Cli/Npm/NpmProvenanceChecker.cs | 40 +++++++++---- .../Agents/NpmProvenanceCheckerTests.cs | 57 +++++++++++++++++++ .../Agents/PlaywrightCliInstallerTests.cs | 2 +- .../TestServices/FakePlaywrightServices.cs | 2 +- 7 files changed, 144 insertions(+), 17 deletions(-) diff --git a/docs/specs/safe-npm-tool-install.md b/docs/specs/safe-npm-tool-install.md index 0beb331bcbb..b2ec266544b 100644 --- a/docs/specs/safe-npm-tool-install.md +++ b/docs/specs/safe-npm-tool-install.md @@ -84,7 +84,7 @@ Our verification chain relies on these trust anchors: | **Source repository** | `predicate.buildDefinition.externalParameters.workflow.repository` | `https://github.com/microsoft/playwright-cli` | The package was built from the legitimate source code | | **Workflow path** | `predicate.buildDefinition.externalParameters.workflow.path` | `.github/workflows/publish.yml` | The build used the expected CI pipeline, not an ad-hoc or attacker-injected workflow | | **Build type** | `predicate.buildDefinition.buildType` | `https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1` | The build ran on GitHub Actions, which implicitly confirms the OIDC token issuer is `https://token.actions.githubusercontent.com` | -| **Workflow ref** | `predicate.buildDefinition.externalParameters.workflow.ref` | `refs/tags/v{version}` (e.g., `refs/tags/v0.1.1`) | The build was triggered from a version tag matching the package version, not an arbitrary branch or commit | +| **Workflow ref** | `predicate.buildDefinition.externalParameters.workflow.ref` | Validated via caller-provided callback (for `@playwright/cli`: kind=`tags`, name=`v{version}`) | The build was triggered from a version tag matching the package version, not an arbitrary branch or commit. The tag format is package-specific — different packages may use different conventions (e.g., `v0.1.1`, `0.1.1`, `@scope/pkg@0.1.1`). The ref is parsed into structured components (`WorkflowRefInfo`) and the caller provides a validation callback. | **What this establishes:** That the Sigstore-attested provenance for this package version claims it was built from the `microsoft/playwright-cli` GitHub repository, using the expected publish workflow, on the GitHub Actions CI system, triggered by a git tag that matches the package version. The build type verification implicitly confirms the OIDC token issuer without needing to parse the Sigstore certificate directly — the SLSA build type `https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1` is only valid for GitHub Actions builds that obtain their signing credentials via GitHub's OIDC provider (`https://token.actions.githubusercontent.com`). The workflow ref verification ensures the build corresponds to an explicit version release, preventing an attacker from publishing a package built from an arbitrary branch or commit. diff --git a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs index af5ba40010c..f13adf8a17f 100644 --- a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs +++ b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs @@ -167,6 +167,8 @@ private async Task InstallCoreAsync(AgentEnvironmentScanContext context, C ExpectedSourceRepository, ExpectedWorkflowPath, ExpectedBuildType, + refInfo => string.Equals(refInfo.Kind, "tags", StringComparison.Ordinal) && + string.Equals(refInfo.Name, $"v{packageInfo.Version}", StringComparison.Ordinal), cancellationToken); if (!provenanceResult.IsVerified) diff --git a/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs b/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs index 151c33145b3..3f9e6aa661b 100644 --- a/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs +++ b/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs @@ -56,7 +56,7 @@ internal enum ProvenanceVerificationOutcome BuildTypeMismatch, /// - /// The workflow ref (git tag) does not match the expected version tag (e.g., refs/tags/v{version}), + /// The workflow ref did not pass the caller-provided validation callback, /// indicating the build was not triggered from the expected release tag. /// WorkflowRefMismatch @@ -117,6 +117,53 @@ internal sealed class ProvenanceVerificationResult public bool IsVerified => Outcome is ProvenanceVerificationOutcome.Verified; } +/// +/// Represents a parsed workflow ref from an SLSA provenance attestation. +/// A workflow ref like refs/tags/v0.1.1 is decomposed into its kind (e.g., "tags") +/// and name (e.g., "v0.1.1") to enable structured validation by callers. +/// +/// The original unmodified ref string (e.g., refs/tags/v0.1.1). +/// The ref kind (e.g., "tags", "heads"). Extracted from the second segment of the ref path. +/// The ref name after the kind prefix (e.g., "v0.1.1", "main"). +internal sealed record WorkflowRefInfo(string Raw, string Kind, string Name) +{ + /// + /// Attempts to parse a git ref string into its structured components. + /// Expected format: refs/{kind}/{name} (e.g., refs/tags/v0.1.1). + /// + /// The raw ref string to parse. + /// The parsed if successful. + /// true if the ref was successfully parsed; false otherwise. + public static bool TryParse(string? refString, out WorkflowRefInfo? refInfo) + { + refInfo = null; + + if (string.IsNullOrEmpty(refString)) + { + return false; + } + + // Expected format: refs/{kind}/{name...} + // The name can contain slashes (e.g., refs/tags/@scope/pkg@1.0.0) + if (!refString.StartsWith("refs/", StringComparison.Ordinal)) + { + return false; + } + + var afterRefs = refString["refs/".Length..]; + var slashIndex = afterRefs.IndexOf('/'); + if (slashIndex <= 0 || slashIndex == afterRefs.Length - 1) + { + return false; + } + + var kind = afterRefs[..slashIndex]; + var name = afterRefs[(slashIndex + 1)..]; + refInfo = new WorkflowRefInfo(refString, kind, name); + return true; + } +} + /// /// Verifies npm package provenance by checking SLSA attestations from the npm registry. /// @@ -131,7 +178,12 @@ internal interface INpmProvenanceChecker /// The expected source repository URL (e.g., "https://github.com/microsoft/playwright-cli"). /// The expected workflow file path (e.g., ".github/workflows/publish.yml"). /// The expected SLSA build type URI identifying the CI system. + /// + /// An optional callback that validates the parsed workflow ref. The callback receives a + /// with the ref decomposed into its kind and name. If null, the workflow ref gate is skipped. + /// If the callback returns false, verification fails with . + /// /// A token to cancel the operation. /// A indicating the outcome and any extracted provenance data. - Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, CancellationToken cancellationToken); + Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, Func? validateWorkflowRef, CancellationToken cancellationToken); } diff --git a/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs b/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs index a95e50335b6..e89b9d449f5 100644 --- a/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs +++ b/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs @@ -16,7 +16,7 @@ internal sealed class NpmProvenanceChecker(HttpClient httpClient, ILogger - public async Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, CancellationToken cancellationToken) + public async Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, Func? validateWorkflowRef, CancellationToken cancellationToken) { // Gate 1: Fetch attestations from the npm registry. string json; @@ -115,20 +115,36 @@ public async Task VerifyProvenanceAsync(string pac }; } - // Gate 6: Verify the workflow ref corresponds to a version tag matching the package version. - var expectedRef = $"refs/tags/v{version}"; - if (!string.Equals(provenance.WorkflowRef, expectedRef, StringComparison.Ordinal)) + // Gate 6: Verify the workflow ref using the caller-provided validation callback. + // Different packages use different tag formats (e.g., "v0.1.1", "0.1.1", "@scope/pkg@0.1.1"), + // so the caller decides what constitutes a valid ref. + if (validateWorkflowRef is not null) { - logger.LogWarning( - "Provenance verification failed: expected workflow ref {Expected} but attestation says {Actual}", - expectedRef, - provenance.WorkflowRef); + if (!WorkflowRefInfo.TryParse(provenance.WorkflowRef, out var refInfo) || refInfo is null) + { + logger.LogWarning( + "Provenance verification failed: could not parse workflow ref {WorkflowRef}", + provenance.WorkflowRef); - return new ProvenanceVerificationResult + return new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.WorkflowRefMismatch, + Provenance = provenance + }; + } + + if (!validateWorkflowRef(refInfo)) { - Outcome = ProvenanceVerificationOutcome.WorkflowRefMismatch, - Provenance = provenance - }; + logger.LogWarning( + "Provenance verification failed: workflow ref {WorkflowRef} did not pass validation", + provenance.WorkflowRef); + + return new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.WorkflowRefMismatch, + Provenance = provenance + }; + } } return new ProvenanceVerificationResult diff --git a/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs b/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs index 2e8b3ce36ba..a6368649911 100644 --- a/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/NpmProvenanceCheckerTests.cs @@ -156,6 +156,8 @@ public async Task VerifyProvenanceAsync_WithMismatchedWorkflowRef_ReturnsWorkflo "https://github.com/microsoft/playwright-cli", ".github/workflows/publish.yml", "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + refInfo => string.Equals(refInfo.Kind, "tags", StringComparison.Ordinal) && + string.Equals(refInfo.Name, "v0.1.1", StringComparison.Ordinal), CancellationToken.None); Assert.Equal(ProvenanceVerificationOutcome.WorkflowRefMismatch, result.Outcome); @@ -179,6 +181,31 @@ public async Task VerifyProvenanceAsync_WithMatchingWorkflowRef_ReturnsVerified( "https://github.com/microsoft/playwright-cli", ".github/workflows/publish.yml", "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + refInfo => string.Equals(refInfo.Kind, "tags", StringComparison.Ordinal) && + string.Equals(refInfo.Name, "v0.1.1", StringComparison.Ordinal), + CancellationToken.None); + + Assert.Equal(ProvenanceVerificationOutcome.Verified, result.Outcome); + } + + [Fact] + public async Task VerifyProvenanceAsync_WithNullCallback_SkipsRefValidation() + { + var json = BuildAttestationJson( + "https://github.com/microsoft/playwright-cli", + workflowRef: "refs/tags/any-format-at-all"); + + var handler = new TestHttpMessageHandler(json); + var httpClient = new HttpClient(handler); + var checker = new NpmProvenanceChecker(httpClient, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + var result = await checker.VerifyProvenanceAsync( + "@playwright/cli", + "0.1.1", + "https://github.com/microsoft/playwright-cli", + ".github/workflows/publish.yml", + "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + validateWorkflowRef: null, CancellationToken.None); Assert.Equal(ProvenanceVerificationOutcome.Verified, result.Outcome); @@ -248,4 +275,34 @@ protected override Task SendAsync(HttpRequestMessage reques }); } } + + [Theory] + [InlineData("refs/tags/v0.1.1", "tags", "v0.1.1")] + [InlineData("refs/heads/main", "heads", "main")] + [InlineData("refs/tags/@scope/pkg@1.0.0", "tags", "@scope/pkg@1.0.0")] + [InlineData("refs/tags/release/1.0.0", "tags", "release/1.0.0")] + public void WorkflowRefInfo_TryParse_ValidRefs_ParsesCorrectly(string raw, string expectedKind, string expectedName) + { + var success = WorkflowRefInfo.TryParse(raw, out var refInfo); + + Assert.True(success); + Assert.NotNull(refInfo); + Assert.Equal(raw, refInfo.Raw); + Assert.Equal(expectedKind, refInfo.Kind); + Assert.Equal(expectedName, refInfo.Name); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("not-a-ref")] + [InlineData("refs/")] + [InlineData("refs/tags/")] + public void WorkflowRefInfo_TryParse_InvalidRefs_ReturnsFalse(string? raw) + { + var success = WorkflowRefInfo.TryParse(raw, out var refInfo); + + Assert.False(success); + Assert.Null(refInfo); + } } diff --git a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs index 7c92fc3cdc9..81b248ee4a3 100644 --- a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs @@ -559,7 +559,7 @@ private sealed class TestNpmProvenanceChecker : INpmProvenanceChecker public ProvenanceVerificationOutcome ProvenanceOutcome { get; set; } = ProvenanceVerificationOutcome.Verified; public bool ProvenanceCalled { get; private set; } - public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, CancellationToken cancellationToken) + public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, Func? validateWorkflowRef, CancellationToken cancellationToken) { ProvenanceCalled = true; return Task.FromResult(new ProvenanceVerificationResult diff --git a/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs b/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs index 4f8e7d1eb3d..b2ea398fc39 100644 --- a/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs +++ b/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs @@ -30,7 +30,7 @@ public Task InstallGlobalAsync(string tarballPath, CancellationToken cance /// internal sealed class FakeNpmProvenanceChecker : INpmProvenanceChecker { - public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, CancellationToken cancellationToken) + public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, Func? validateWorkflowRef, CancellationToken cancellationToken) => Task.FromResult(new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.Verified, From 79e6995b0fbd43ae6f021baf5c5664543956576a Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 3 Mar 2026 14:13:42 +1100 Subject: [PATCH 13/15] Replace npm provenance checking with Sigstore verification Use the Sigstore and Tuf NuGet packages (0.2.0) to cryptographically verify npm attestation bundles in-process, replacing the previous approach of shelling out to 'npm audit signatures'. - Add SigstoreNpmProvenanceChecker implementing INpmProvenanceChecker using SigstoreVerifier with CertificateIdentity.ForGitHubActions - Remove the npm audit signatures step from PlaywrightCliInstaller - Keep existing NpmProvenanceChecker but no longer register in DI - Add optional sriIntegrity parameter to INpmProvenanceChecker for digest-based bundle verification - Update safe-npm-tool-install.md spec to reflect new verification flow - Temporarily add nuget.org to NuGet.config for Sigstore/Tuf packages Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Directory.Packages.props | 2 + NuGet.config | 8 + docs/specs/safe-npm-tool-install.md | 94 ++---- .../Playwright/PlaywrightCliInstaller.cs | 30 +- src/Aspire.Cli/Aspire.Cli.csproj | 5 + src/Aspire.Cli/Npm/INpmProvenanceChecker.cs | 7 +- src/Aspire.Cli/Npm/NpmProvenanceChecker.cs | 2 +- .../Npm/SigstoreNpmProvenanceChecker.cs | 301 ++++++++++++++++++ src/Aspire.Cli/Program.cs | 2 +- .../Agents/PlaywrightCliInstallerTests.cs | 31 +- .../SigstoreNpmProvenanceCheckerTests.cs | 161 ++++++++++ .../Aspire.Cli.Tests/Aspire.Cli.Tests.csproj | 1 + .../TestServices/FakePlaywrightServices.cs | 2 +- 13 files changed, 531 insertions(+), 115 deletions(-) create mode 100644 src/Aspire.Cli/Npm/SigstoreNpmProvenanceChecker.cs create mode 100644 tests/Aspire.Cli.Tests/Agents/SigstoreNpmProvenanceCheckerTests.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 4f04edc0fb1..a6fee1a26d2 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -131,6 +131,8 @@ + + diff --git a/NuGet.config b/NuGet.config index 2822bc5d52a..c923efacf56 100644 --- a/NuGet.config +++ b/NuGet.config @@ -20,6 +20,8 @@ + + @@ -43,6 +45,12 @@ + + + + + + diff --git a/docs/specs/safe-npm-tool-install.md b/docs/specs/safe-npm-tool-install.md index b2ec266544b..ab4367bb022 100644 --- a/docs/specs/safe-npm-tool-install.md +++ b/docs/specs/safe-npm-tool-install.md @@ -27,7 +27,7 @@ Our verification chain relies on these trust anchors: | Trust anchor | What it provides | How it's protected | |---|---|---| | **npm registry** | Package metadata, tarball hosting | HTTPS/TLS, npm's infrastructure security | -| **Sigstore (Fulcio + Rekor)** | Cryptographic attestation signatures | Public CA with OIDC federation, append-only transparency log | +| **Sigstore (Fulcio + Rekor)** | Cryptographic attestation signatures | Public CA with OIDC federation, append-only transparency log, verified in-process via Sigstore .NET library with TUF trust root | | **GitHub Actions OIDC** | Builder identity claims in Sigstore certificates | GitHub's infrastructure security | | **Hardcoded expected values** | Package name, version range, expected source repository | Code review, our own release process | @@ -51,33 +51,15 @@ Our verification chain relies on these trust anchors: **Trust basis:** The previously-installed binary. If the user's system is compromised, this could be spoofed, but that's outside our threat model. -### Step 3: Verify Sigstore attestations via npm - -**Action:** -1. Create a temporary directory with a minimal `package.json` -2. Run `npm install @playwright/cli@{version} --ignore-scripts` to install the package from the registry as a project dependency -3. Run `npm audit signatures` to verify Sigstore attestation signatures - -**What this establishes:** That valid Sigstore-signed attestations exist for `@playwright/cli@{version}`. Specifically: - -- The npm registry has attestation bundles for this package version -- The attestation signatures are cryptographically valid (signed by Sigstore's Fulcio CA) -- The attestation entries are present in the Rekor transparency log (inclusion proof verified) -- The OIDC identity in the signing certificate corresponds to a GitHub Actions workflow - -**Trust basis:** Sigstore's public key infrastructure. Even if the npm registry is compromised, an attacker cannot forge valid Sigstore signatures — they would need to compromise Fulcio (the Sigstore CA) or obtain a valid OIDC token from GitHub Actions for the legitimate repository's workflow. - -**Why a temporary project is needed:** `npm audit signatures` operates on installed project dependencies. It requires `node_modules` and a `package-lock.json` to know which packages to verify. For a global tool install there is no project context, so we create one temporarily. The package must be installed from the registry (not from a local tarball) because `npm audit signatures` skips packages with `resolved: file:...` in the lockfile. - -**Limitations:** `npm audit signatures` verifies that *valid attestations exist* but does not expose the attestation *content*. It confirms "this package has authentic Sigstore-signed attestations" but does not tell us *what* those attestations say (e.g., which repository built the package). That's addressed in Step 4. - -### Step 4: Verify provenance metadata +### Step 3: Verify Sigstore attestation and provenance metadata **Action:** 1. Fetch the attestation bundle from `https://registry.npmjs.org/-/npm/v1/attestations/@playwright/cli@{version}` 2. Find the attestation with `predicateType: "https://slsa.dev/provenance/v1"` (SLSA Build L3 provenance) -3. Base64-decode the DSSE envelope payload to extract the in-toto statement -4. Verify the following fields from the provenance predicate: +3. Extract the Sigstore bundle from the `bundle` field of the attestation +4. Cryptographically verify the Sigstore bundle using the `SigstoreVerifier` from the [Sigstore .NET library](https://github.com/mitchdenny/sigstore-dotnet), with a `VerificationPolicy` configured for `CertificateIdentity.ForGitHubActions("microsoft", "playwright-cli")` +5. Base64-decode the DSSE envelope payload to extract the in-toto statement +6. Verify the following fields from the provenance predicate: | Field | Location in payload | Expected value | What it proves | |---|---|---|---| @@ -86,19 +68,15 @@ Our verification chain relies on these trust anchors: | **Build type** | `predicate.buildDefinition.buildType` | `https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1` | The build ran on GitHub Actions, which implicitly confirms the OIDC token issuer is `https://token.actions.githubusercontent.com` | | **Workflow ref** | `predicate.buildDefinition.externalParameters.workflow.ref` | Validated via caller-provided callback (for `@playwright/cli`: kind=`tags`, name=`v{version}`) | The build was triggered from a version tag matching the package version, not an arbitrary branch or commit. The tag format is package-specific — different packages may use different conventions (e.g., `v0.1.1`, `0.1.1`, `@scope/pkg@0.1.1`). The ref is parsed into structured components (`WorkflowRefInfo`) and the caller provides a validation callback. | -**What this establishes:** That the Sigstore-attested provenance for this package version claims it was built from the `microsoft/playwright-cli` GitHub repository, using the expected publish workflow, on the GitHub Actions CI system, triggered by a git tag that matches the package version. The build type verification implicitly confirms the OIDC token issuer without needing to parse the Sigstore certificate directly — the SLSA build type `https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1` is only valid for GitHub Actions builds that obtain their signing credentials via GitHub's OIDC provider (`https://token.actions.githubusercontent.com`). The workflow ref verification ensures the build corresponds to an explicit version release, preventing an attacker from publishing a package built from an arbitrary branch or commit. +**What this establishes:** That the Sigstore bundle is cryptographically authentic — the signing certificate was issued by Sigstore's Fulcio CA, the signature is recorded in the Rekor transparency log, and the OIDC identity in the certificate matches the `microsoft/playwright-cli` GitHub Actions workflow. Additionally, the provenance metadata confirms the package was built from the expected repository, workflow, CI system, and version tag. -**Additional fields extracted but not directly verified:** The provenance parser also extracts `runDetails.builder.id` from the attestation. This is available in the `NpmProvenanceData` result for logging and diagnostics but is not currently used as a verification gate. - -**Trust basis:** The attestation content is protected by the Sigstore signature verified in Step 3. Since Step 3 confirmed the attestation is cryptographically authentic (signed by a valid Sigstore certificate corresponding to a GitHub Actions OIDC identity), the content we read in Step 4 cannot have been tampered with by the npm registry. An attacker would need to compromise Sigstore itself to forge attestation content pointing to `microsoft/playwright-cli` with the correct workflow and build type. +**Trust basis:** Sigstore's public key infrastructure via the `Sigstore` and `Tuf` .NET libraries. The TUF trust root is automatically downloaded and verified. Even if the npm registry is compromised, an attacker cannot forge valid Sigstore signatures — they would need to compromise Fulcio (the Sigstore CA) or obtain a valid OIDC token from GitHub Actions for the legitimate repository's workflow. Since the Sigstore verification and provenance field checking happen on the same attestation bundle in a single operation, there is no TOCTOU gap between signature verification and content inspection. -**Why we verify all three fields:** Checking only the source repository would leave a gap where an attacker with write access to the repo could introduce a malicious workflow (e.g., `.github/workflows/evil.yml`) that builds a tampered package. By also verifying the workflow path and build type, we ensure the package was built by the specific, expected CI pipeline running on the expected CI system. +**Why we verify all provenance fields:** Checking only the Sigstore certificate identity (GitHub Actions + repository) is necessary but not sufficient. An attacker with write access to the repo could introduce a malicious workflow (e.g., `.github/workflows/evil.yml`). By also verifying the workflow path, build type, and workflow ref, we ensure the package was built by the specific expected CI pipeline from a release tag. -**Why we fetch from the registry API:** The npm CLI (`npm audit signatures`) verifies Sigstore signatures but does not expose provenance content in its output. The `--json` flag only produces `{"invalid":[],"missing":[]}`. There is no npm CLI command to read the source repository from a SLSA provenance attestation. We must fetch the attestation bundle directly from the registry API. - -**Note on reading attested content from an untrusted source:** We are reading the attestation JSON from the same npm registry that could theoretically be compromised. However, the attestation *signature* was already verified by `npm audit signatures` in Step 3 using Sigstore's independent trust chain. We are relying on the fact that the registry serves the same attestation bundle that npm verified. If the registry served different attestation data to our HTTP request than what `npm audit signatures` verified, the provenance content could be spoofed. This is a residual risk — see "Residual Risks" below. +**Additional fields extracted but not directly verified:** The provenance parser also extracts `runDetails.builder.id` from the attestation. This is available in the `NpmProvenanceData` result for logging and diagnostics but is not currently used as a verification gate. -### Step 5: Download and verify tarball integrity +### Step 4: Download and verify tarball integrity **Action:** 1. Run `npm pack @playwright/cli@{version}` to download the tarball @@ -111,15 +89,15 @@ Our verification chain relies on these trust anchors: **Relationship to Step 3:** The Sigstore attestations verified in Step 3 are bound to the package version and its published content. The integrity hash in the registry packument is the canonical identifier for the tarball content. By verifying our tarball matches this hash, we establish that our tarball is the same artifact that the Sigstore attestations cover. -### Step 6: Install globally from verified tarball +### Step 5: Install globally from verified tarball **Action:** Run `npm install -g {tarballPath}` to install the verified tarball as a global tool. **What this establishes:** The tool is installed and available on the user's PATH. -**Trust basis:** All preceding verification steps have passed. The tarball content has been verified against the registry's published hash (Step 5), the Sigstore attestations for that content are cryptographically valid (Step 3), and the attestations claim the correct source repository (Step 4). +**Trust basis:** All preceding verification steps have passed. The tarball content has been verified against the registry's published hash (Step 4), the Sigstore attestations for that content are cryptographically valid (Step 3), and the attestations confirm the correct source repository, workflow, and build system (Step 3). -### Step 7: Generate and mirror skill files +### Step 6: Generate and mirror skill files **Action:** Run `playwright-cli install --skills` to generate agent skill files in the primary skill directory (`.claude/skills/playwright-cli/`), then mirror the skill directory to all other detected agent environment skill directories (e.g., `.github/skills/playwright-cli/`, `.opencode/skill/playwright-cli/`). The mirror is a full sync — files are created, updated, and stale files are removed so all environments have identical skill content. @@ -146,42 +124,34 @@ Our verification chain relies on these trust anchors: └──────────────┬────────────────┘ │ ┌────────────────────┼────────────────────┐ - │ │ │ - ┌──────────▼──────────┐ ┌─────▼──────────┐ ┌──────▼──────────┐ - │ Step 3: npm audit │ │ Step 4: Verify │ │ Step 5: npm pack│ - │ signatures │ │ provenance │ │ + SHA-512 check │ - │ (Sigstore crypto) │ │ (repo, workflow │ │ (tarball │ - │ │ │ + build type) │ │ integrity) │ - └──────────┬───────────┘ └─────┬──────────┘ └──────┬──────────┘ - │ │ │ - │ Attestation is │ Built from │ Tarball matches - │ authentic │ expected repo + │ published hash - │ │ expected pipeline │ - └────────────────────┼─────────────────────┘ + │ │ + ┌──────────▼──────────────┐ ┌─────────▼─────────┐ + │ Step 3: Sigstore verify │ │ Step 4: npm pack │ + │ + provenance checks │ │ + SHA-512 check │ + │ (in-process via Sigstore │ │ (tarball │ + │ .NET library + TUF) │ │ integrity) │ + └──────────┬───────────────┘ └─────────┬─────────┘ + │ │ + │ Attestation is authentic + │ Tarball matches + │ built from expected repo + │ published hash + │ expected pipeline │ + └────────────────────┬────────────────────┘ │ ┌──────────────▼────────────────┐ - │ Step 6: npm install -g │ + │ Step 5: npm install -g │ │ (from verified tarball) │ └───────────────────────────────┘ ``` ## Residual Risks -### 1. Registry serving different attestation data to different clients - -**Risk:** The npm registry could theoretically serve one attestation bundle to `npm audit signatures` (which passes verification) and a different bundle to our HTTP API request (with spoofed provenance content). - -**Mitigation:** This would require active, targeted compromise of the npm registry's serving infrastructure — not just a publish token theft or package tampering. The Rekor transparency log provides a public record of all attestations, making such targeted serving detectable. - -**Alternative mitigation:** We could eliminate this risk entirely by parsing the attestation bundle ourselves and verifying the Sigstore signature directly in C#. This would require implementing ECDSA signature verification, X.509 certificate chain validation, and Merkle inclusion proof verification. This significantly increases implementation complexity and is not recommended for the initial implementation. - -### 2. Time-of-check-to-time-of-use (TOCTOU) +### 1. Time-of-check-to-time-of-use (TOCTOU) **Risk:** The package could be replaced on the registry between our verification steps and the global install. -**Mitigation:** We verify the SHA-512 hash of the tarball we actually install (Step 5), and we install from the local tarball file (not from the registry again). The verified tarball is the same file that gets installed. +**Mitigation:** We verify the SHA-512 hash of the tarball we actually install (Step 4), and we install from the local tarball file (not from the registry again). The verified tarball is the same file that gets installed. -### 3. Transitive dependency attacks +### 2. Transitive dependency attacks **Risk:** `@playwright/cli` has dependencies that could be compromised. @@ -210,6 +180,4 @@ Two break-glass configuration keys are available via `aspire config set`: ## Future Improvements -1. **Direct Sigstore verification in .NET** — An experimental implementation exists on the `sigstore-builtin-verification` branch using the [`Sigstore`](https://github.com/mitchdenny/sigstore-dotnet) .NET library. This eliminates residual risk #1 by performing Sigstore bundle verification natively (Fulcio certificate chain, Rekor transparency log inclusion proof, SCT verification, DSSE signature verification) without relying on `npm audit signatures`. It is gated behind the `builtInSigstoreVerificationEnabled` feature flag. -2. **Certificate identity verification** — The experimental Sigstore path already verifies the OIDC identity claims in the Sigstore signing certificate (issuer and SAN) via `CertificateIdentity.ForGitHubActions(repository)`. Bringing this to the default path would provide defense-in-depth beyond the provenance payload checks. -3. **Pinned tarball hash** — Ship a known-good SRI hash with each Aspire release, eliminating the need to trust the registry for the hash at all. +1. **Pinned tarball hash** — Ship a known-good SRI hash with each Aspire release, eliminating the need to trust the registry for the hash at all. diff --git a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs index f13adf8a17f..23dcfb35060 100644 --- a/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs +++ b/src/Aspire.Cli/Agents/Playwright/PlaywrightCliInstaller.cs @@ -144,22 +144,9 @@ private async Task InstallCoreAsync(AgentEnvironmentScanContext context, C if (!validationDisabled) { - // Step 3: Verify Sigstore attestation signatures via npm audit signatures. - // This creates a temporary project to give npm audit signatures a working context. - logger.LogDebug("Verifying Sigstore attestations for {Package}@{Version}", PackageName, packageInfo.Version); - var auditPassed = await npmRunner.AuditSignaturesAsync(PackageName, packageInfo.Version.ToString(), cancellationToken); - if (!auditPassed) - { - logger.LogWarning( - "Sigstore attestation verification failed for {Package}@{Version}. The package may not have valid attestations.", - PackageName, - packageInfo.Version); - return false; - } - - logger.LogDebug("Sigstore attestation verification passed for {Package}@{Version}", PackageName, packageInfo.Version); - - // Step 4: Verify provenance source repository from SLSA attestation. + // Step 3: Verify provenance via Sigstore bundle verification and SLSA attestation checks. + // This cryptographically verifies the Sigstore bundle (Fulcio CA, Rekor tlog, OIDC identity) + // and then checks the provenance fields (source repo, workflow, build type, ref). logger.LogDebug("Verifying provenance for {Package}@{Version}", PackageName, packageInfo.Version); var provenanceResult = await provenanceChecker.VerifyProvenanceAsync( PackageName, @@ -169,7 +156,8 @@ private async Task InstallCoreAsync(AgentEnvironmentScanContext context, C ExpectedBuildType, refInfo => string.Equals(refInfo.Kind, "tags", StringComparison.Ordinal) && string.Equals(refInfo.Name, $"v{packageInfo.Version}", StringComparison.Ordinal), - cancellationToken); + cancellationToken, + sriIntegrity: packageInfo.Integrity); if (!provenanceResult.IsVerified) { @@ -189,7 +177,7 @@ private async Task InstallCoreAsync(AgentEnvironmentScanContext context, C provenanceResult.Provenance?.SourceRepository); } - // Step 5: Download the tarball via npm pack. + // Step 4: Download the tarball via npm pack. var tempDir = Path.Combine(Path.GetTempPath(), $"aspire-playwright-{Guid.NewGuid():N}"); Directory.CreateDirectory(tempDir); @@ -204,7 +192,7 @@ private async Task InstallCoreAsync(AgentEnvironmentScanContext context, C return false; } - // Step 6: Verify the downloaded tarball's SHA-512 hash matches the SRI integrity value. + // Step 5: Verify the downloaded tarball's SHA-512 hash matches the SRI integrity value. if (!validationDisabled && !VerifyIntegrity(tarballPath, packageInfo.Integrity)) { logger.LogWarning( @@ -219,7 +207,7 @@ private async Task InstallCoreAsync(AgentEnvironmentScanContext context, C logger.LogDebug("Integrity verification passed for {TarballPath}", tarballPath); } - // Step 7: Install globally from the verified tarball. + // Step 6: Install globally from the verified tarball. logger.LogDebug("Installing {Package}@{Version} globally", PackageName, packageInfo.Version); var installSuccess = await npmRunner.InstallGlobalAsync(tarballPath, cancellationToken); @@ -229,7 +217,7 @@ private async Task InstallCoreAsync(AgentEnvironmentScanContext context, C return false; } - // Step 8: Generate skill files. + // Step 7: Generate skill files. logger.LogDebug("Generating Playwright CLI skill files"); var skillsResult = await playwrightCliRunner.InstallSkillsAsync(cancellationToken); if (skillsResult) diff --git a/src/Aspire.Cli/Aspire.Cli.csproj b/src/Aspire.Cli/Aspire.Cli.csproj index 8eedc788dfd..08a9a457e86 100644 --- a/src/Aspire.Cli/Aspire.Cli.csproj +++ b/src/Aspire.Cli/Aspire.Cli.csproj @@ -16,6 +16,9 @@ in BackchannelJsonSerializerContext.cs. Suppress until MCP graduates these types. --> $(NoWarn);CS1591;MCPEXP001 true + + false false Size $(DefineConstants);CLI @@ -54,6 +57,8 @@ + + diff --git a/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs b/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs index 3f9e6aa661b..6b3a7616c24 100644 --- a/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs +++ b/src/Aspire.Cli/Npm/INpmProvenanceChecker.cs @@ -184,6 +184,11 @@ internal interface INpmProvenanceChecker /// If the callback returns false, verification fails with . /// /// A token to cancel the operation. + /// + /// An optional SRI integrity string (e.g., "sha512-...") for the package tarball. + /// When provided, implementations that perform cryptographic verification can verify + /// that the attestation covers this specific artifact digest. + /// /// A indicating the outcome and any extracted provenance data. - Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, Func? validateWorkflowRef, CancellationToken cancellationToken); + Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, Func? validateWorkflowRef, CancellationToken cancellationToken, string? sriIntegrity = null); } diff --git a/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs b/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs index e89b9d449f5..423b5e1e686 100644 --- a/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs +++ b/src/Aspire.Cli/Npm/NpmProvenanceChecker.cs @@ -16,7 +16,7 @@ internal sealed class NpmProvenanceChecker(HttpClient httpClient, ILogger - public async Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, Func? validateWorkflowRef, CancellationToken cancellationToken) + public async Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, Func? validateWorkflowRef, CancellationToken cancellationToken, string? sriIntegrity = null) { // Gate 1: Fetch attestations from the npm registry. string json; diff --git a/src/Aspire.Cli/Npm/SigstoreNpmProvenanceChecker.cs b/src/Aspire.Cli/Npm/SigstoreNpmProvenanceChecker.cs new file mode 100644 index 00000000000..cad83094f1c --- /dev/null +++ b/src/Aspire.Cli/Npm/SigstoreNpmProvenanceChecker.cs @@ -0,0 +1,301 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Nodes; +using Microsoft.Extensions.Logging; +using Sigstore; + +namespace Aspire.Cli.Npm; + +/// +/// Verifies npm package provenance by cryptographically verifying Sigstore bundles +/// from the npm registry attestations API using the Sigstore .NET library. +/// +internal sealed class SigstoreNpmProvenanceChecker(HttpClient httpClient, ILogger logger) : INpmProvenanceChecker +{ + internal const string NpmRegistryAttestationsBaseUrl = "https://registry.npmjs.org/-/npm/v1/attestations"; + internal const string SlsaProvenancePredicateType = "https://slsa.dev/provenance/v1"; + + /// + public async Task VerifyProvenanceAsync( + string packageName, + string version, + string expectedSourceRepository, + string expectedWorkflowPath, + string expectedBuildType, + Func? validateWorkflowRef, + CancellationToken cancellationToken, + string? sriIntegrity = null) + { + // Gate 1: Fetch attestations from the npm registry. + string json; + try + { + var encodedPackage = Uri.EscapeDataString(packageName); + var url = $"{NpmRegistryAttestationsBaseUrl}/{encodedPackage}@{version}"; + + logger.LogDebug("Fetching attestations from {Url}", url); + var response = await httpClient.GetAsync(url, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + logger.LogDebug("Failed to fetch attestations: HTTP {StatusCode}", response.StatusCode); + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationFetchFailed }; + } + + json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException ex) + { + logger.LogDebug(ex, "Failed to fetch attestations for {Package}@{Version}", packageName, version); + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationFetchFailed }; + } + + // Gate 2: Find the SLSA provenance attestation and extract its Sigstore bundle. + JsonNode? bundleNode; + try + { + bundleNode = FindSlsaProvenanceBundle(json); + if (bundleNode is null) + { + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.SlsaProvenanceNotFound }; + } + } + catch (Exception ex) when (ex is System.Text.Json.JsonException or InvalidOperationException) + { + logger.LogDebug(ex, "Failed to parse attestation response for {Package}@{Version}", packageName, version); + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationParseFailed }; + } + + // Gate 3: Cryptographically verify the Sigstore bundle using the Sigstore library. + // This verifies the Fulcio certificate chain, Rekor transparency log inclusion, and OIDC identity. + var bundleJson = bundleNode.ToJsonString(); + SigstoreBundle bundle; + try + { + bundle = SigstoreBundle.Deserialize(bundleJson); + } + catch (Exception ex) + { + logger.LogDebug(ex, "Failed to deserialize Sigstore bundle for {Package}@{Version}", packageName, version); + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationParseFailed }; + } + + // Extract the owner and repo from the expected source repository URL. + // Expected format: "https://github.com/{owner}/{repo}" + if (!TryParseGitHubOwnerRepo(expectedSourceRepository, out var owner, out var repo)) + { + logger.LogWarning("Could not parse GitHub owner/repo from expected source repository: {ExpectedSourceRepository}", expectedSourceRepository); + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.SourceRepositoryNotFound }; + } + + var verifier = new SigstoreVerifier(); + var policy = new VerificationPolicy + { + CertificateIdentity = CertificateIdentity.ForGitHubActions(owner, repo) + }; + + try + { + bool success; + VerificationResult? result; + + if (sriIntegrity is not null && sriIntegrity.StartsWith("sha512-", StringComparison.OrdinalIgnoreCase)) + { + // Verify the bundle against the tarball's SHA-512 digest from the SRI integrity string. + var hashBase64 = sriIntegrity["sha512-".Length..]; + var digestBytes = Convert.FromBase64String(hashBase64); + + (success, result) = await verifier.TryVerifyDigestAsync( + digestBytes, HashAlgorithmType.Sha512, bundle, policy).ConfigureAwait(false); + } + else + { + // No integrity hash available — verify using the DSSE envelope payload bytes. + // The DSSE payload is the in-toto statement that was signed. + var payloadNode = bundleNode["dsseEnvelope"]?["payload"]?.GetValue(); + if (payloadNode is null) + { + logger.LogDebug("No DSSE payload found in bundle for {Package}@{Version}", packageName, version); + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.PayloadDecodeFailed }; + } + + var payloadBytes = Convert.FromBase64String(payloadNode); + (success, result) = await verifier.TryVerifyAsync( + payloadBytes, bundle, policy).ConfigureAwait(false); + } + + if (!success) + { + logger.LogWarning( + "Sigstore verification failed for {Package}@{Version}: {FailureReason}", + packageName, version, result?.FailureReason); + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationParseFailed }; + } + + logger.LogDebug( + "Sigstore verification passed for {Package}@{Version}. Signed by: {Signer}", + packageName, version, result?.SignerIdentity?.SubjectAlternativeName); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Sigstore verification threw an exception for {Package}@{Version}", packageName, version); + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationParseFailed }; + } + + // Gate 4: Parse the DSSE envelope payload for provenance data and apply field-level checks. + NpmProvenanceData provenance; + try + { + var parseResult = NpmProvenanceChecker.ParseProvenance(json); + if (parseResult is null) + { + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.SlsaProvenanceNotFound }; + } + + provenance = parseResult.Value.Provenance; + if (parseResult.Value.Outcome is not ProvenanceVerificationOutcome.Verified) + { + return new ProvenanceVerificationResult + { + Outcome = parseResult.Value.Outcome, + Provenance = provenance + }; + } + } + catch (System.Text.Json.JsonException ex) + { + logger.LogDebug(ex, "Failed to parse provenance data from attestation for {Package}@{Version}", packageName, version); + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationParseFailed }; + } + + logger.LogDebug("SLSA provenance source repository: {SourceRepository}", provenance.SourceRepository); + + // Gate 5: Verify the source repository matches. + if (!string.Equals(provenance.SourceRepository, expectedSourceRepository, StringComparison.OrdinalIgnoreCase)) + { + logger.LogWarning( + "Provenance verification failed: expected source repository {Expected} but attestation says {Actual}", + expectedSourceRepository, provenance.SourceRepository); + + return new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.SourceRepositoryMismatch, + Provenance = provenance + }; + } + + // Gate 6: Verify the workflow path matches. + if (!string.Equals(provenance.WorkflowPath, expectedWorkflowPath, StringComparison.Ordinal)) + { + logger.LogWarning( + "Provenance verification failed: expected workflow path {Expected} but attestation says {Actual}", + expectedWorkflowPath, provenance.WorkflowPath); + + return new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.WorkflowMismatch, + Provenance = provenance + }; + } + + // Gate 7: Verify the build type matches. + if (!string.Equals(provenance.BuildType, expectedBuildType, StringComparison.Ordinal)) + { + logger.LogWarning( + "Provenance verification failed: expected build type {Expected} but attestation says {Actual}", + expectedBuildType, provenance.BuildType); + + return new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.BuildTypeMismatch, + Provenance = provenance + }; + } + + // Gate 8: Verify the workflow ref using the caller-provided validation callback. + if (validateWorkflowRef is not null) + { + if (!WorkflowRefInfo.TryParse(provenance.WorkflowRef, out var refInfo) || refInfo is null) + { + logger.LogWarning( + "Provenance verification failed: could not parse workflow ref {WorkflowRef}", + provenance.WorkflowRef); + + return new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.WorkflowRefMismatch, + Provenance = provenance + }; + } + + if (!validateWorkflowRef(refInfo)) + { + logger.LogWarning( + "Provenance verification failed: workflow ref {WorkflowRef} did not pass validation", + provenance.WorkflowRef); + + return new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.WorkflowRefMismatch, + Provenance = provenance + }; + } + } + + return new ProvenanceVerificationResult + { + Outcome = ProvenanceVerificationOutcome.Verified, + Provenance = provenance + }; + } + + /// + /// Finds the Sigstore bundle JSON node for the SLSA provenance attestation. + /// + internal static JsonNode? FindSlsaProvenanceBundle(string attestationJson) + { + var doc = JsonNode.Parse(attestationJson); + var attestations = doc?["attestations"]?.AsArray(); + + if (attestations is null || attestations.Count == 0) + { + return null; + } + + foreach (var attestation in attestations) + { + var predicateType = attestation?["predicateType"]?.GetValue(); + if (string.Equals(predicateType, SlsaProvenancePredicateType, StringComparison.Ordinal)) + { + return attestation?["bundle"]; + } + } + + return null; + } + + /// + /// Parses a GitHub repository URL into owner and repo components. + /// + internal static bool TryParseGitHubOwnerRepo(string repositoryUrl, out string owner, out string repo) + { + owner = string.Empty; + repo = string.Empty; + + if (!Uri.TryCreate(repositoryUrl, UriKind.Absolute, out var uri)) + { + return false; + } + + var segments = uri.AbsolutePath.Trim('/').Split('/'); + if (segments.Length < 2) + { + return false; + } + + owner = segments[0]; + repo = segments[1]; + return true; + } +} diff --git a/src/Aspire.Cli/Program.cs b/src/Aspire.Cli/Program.cs index 01991026ad1..3f633a3e74c 100644 --- a/src/Aspire.Cli/Program.cs +++ b/src/Aspire.Cli/Program.cs @@ -328,7 +328,7 @@ internal static async Task BuildApplicationAsync(string[] args, Dictionar // Npm and Playwright CLI operations. builder.Services.AddSingleton(); - builder.Services.AddHttpClient(); + builder.Services.AddHttpClient(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs index 81b248ee4a3..ee4f1f9f619 100644 --- a/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/PlaywrightCliInstallerTests.cs @@ -155,8 +155,7 @@ public async Task InstallAsync_WhenIntegrityCheckPasses_InstallsGlobally() { ResolveResult = new NpmPackageInfo { Version = version, Integrity = integrity }, PackResult = tarballPath, - InstallGlobalResult = true, - AuditResult = true + InstallGlobalResult = true }; var playwrightRunner = new TestPlaywrightCliRunner { @@ -230,8 +229,7 @@ public async Task InstallAsync_WhenOlderVersionInstalled_PerformsUpgrade() { ResolveResult = new NpmPackageInfo { Version = targetVersion, Integrity = integrity }, PackResult = tarballPath, - InstallGlobalResult = true, - AuditResult = true + InstallGlobalResult = true }; var playwrightRunner = new TestPlaywrightCliRunner { @@ -305,33 +303,13 @@ public void VerifyIntegrity_WithNonSha512Prefix_ReturnsFalse() } } - [Fact] - public async Task InstallAsync_WhenAuditSignaturesFails_ReturnsFalse() - { - var version = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); - var npmRunner = new TestNpmRunner - { - ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-abc123" }, - AuditResult = false - }; - var provenanceChecker = new TestNpmProvenanceChecker(); - var playwrightRunner = new TestPlaywrightCliRunner(); - var installer = new PlaywrightCliInstaller(npmRunner, provenanceChecker, playwrightRunner, new TestConsoleInteractionService(), new ConfigurationBuilder().Build(), NullLogger.Instance); - - var result = await installer.InstallAsync(CreateTestContext(), CancellationToken.None); - - Assert.False(result); - Assert.False(provenanceChecker.ProvenanceCalled); - } - [Fact] public async Task InstallAsync_WhenProvenanceCheckFails_ReturnsFalse() { var version = SemVersion.Parse("0.1.1", SemVersionStyles.Strict); var npmRunner = new TestNpmRunner { - ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-abc123" }, - AuditResult = true + ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-abc123" } }; var provenanceChecker = new TestNpmProvenanceChecker { ProvenanceOutcome = ProvenanceVerificationOutcome.SourceRepositoryMismatch }; var playwrightRunner = new TestPlaywrightCliRunner(); @@ -359,7 +337,6 @@ public async Task InstallAsync_WhenValidationDisabled_SkipsAllValidationChecks() var npmRunner = new TestNpmRunner { ResolveResult = new NpmPackageInfo { Version = version, Integrity = "sha512-wronghash" }, - AuditResult = false, PackResult = tarballPath }; var provenanceChecker = new TestNpmProvenanceChecker { ProvenanceOutcome = ProvenanceVerificationOutcome.AttestationFetchFailed }; @@ -559,7 +536,7 @@ private sealed class TestNpmProvenanceChecker : INpmProvenanceChecker public ProvenanceVerificationOutcome ProvenanceOutcome { get; set; } = ProvenanceVerificationOutcome.Verified; public bool ProvenanceCalled { get; private set; } - public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, Func? validateWorkflowRef, CancellationToken cancellationToken) + public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, Func? validateWorkflowRef, CancellationToken cancellationToken, string? sriIntegrity = null) { ProvenanceCalled = true; return Task.FromResult(new ProvenanceVerificationResult diff --git a/tests/Aspire.Cli.Tests/Agents/SigstoreNpmProvenanceCheckerTests.cs b/tests/Aspire.Cli.Tests/Agents/SigstoreNpmProvenanceCheckerTests.cs new file mode 100644 index 00000000000..b4cc08f4e21 --- /dev/null +++ b/tests/Aspire.Cli.Tests/Agents/SigstoreNpmProvenanceCheckerTests.cs @@ -0,0 +1,161 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Npm; + +namespace Aspire.Cli.Tests.Agents; + +public class SigstoreNpmProvenanceCheckerTests +{ + [Fact] + public void FindSlsaProvenanceBundle_WithValidSlsaAttestation_ReturnsBundle() + { + var json = BuildAttestationJsonWithBundle("https://github.com/microsoft/playwright-cli"); + + var bundle = SigstoreNpmProvenanceChecker.FindSlsaProvenanceBundle(json); + + Assert.NotNull(bundle); + Assert.NotNull(bundle["dsseEnvelope"]); + } + + [Fact] + public void FindSlsaProvenanceBundle_WithNoSlsaPredicate_ReturnsNull() + { + var json = """ + { + "attestations": [ + { + "predicateType": "https://github.com/npm/attestation/tree/main/specs/publish/v0.1", + "bundle": { + "dsseEnvelope": { + "payload": "" + } + } + } + ] + } + """; + + var bundle = SigstoreNpmProvenanceChecker.FindSlsaProvenanceBundle(json); + + Assert.Null(bundle); + } + + [Fact] + public void FindSlsaProvenanceBundle_WithEmptyAttestations_ReturnsNull() + { + var json = """{"attestations": []}"""; + + var bundle = SigstoreNpmProvenanceChecker.FindSlsaProvenanceBundle(json); + + Assert.Null(bundle); + } + + [Theory] + [InlineData("https://github.com/microsoft/playwright-cli", "microsoft", "playwright-cli")] + [InlineData("https://github.com/dotnet/aspire", "dotnet", "aspire")] + [InlineData("https://github.com/owner/repo", "owner", "repo")] + public void TryParseGitHubOwnerRepo_WithValidUrl_ReturnsTrueAndParsesComponents(string url, string expectedOwner, string expectedRepo) + { + var result = SigstoreNpmProvenanceChecker.TryParseGitHubOwnerRepo(url, out var owner, out var repo); + + Assert.True(result); + Assert.Equal(expectedOwner, owner); + Assert.Equal(expectedRepo, repo); + } + + [Theory] + [InlineData("not-a-url")] + [InlineData("https://github.com/")] + [InlineData("https://github.com/only-owner")] + public void TryParseGitHubOwnerRepo_WithInvalidUrl_ReturnsFalse(string url) + { + var result = SigstoreNpmProvenanceChecker.TryParseGitHubOwnerRepo(url, out _, out _); + + Assert.False(result); + } + + private static string BuildAttestationJsonWithBundle(string sourceRepository) + { + var payload = BuildProvenancePayload(sourceRepository); + var payloadBase64 = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(payload)); + + return $$""" + { + "attestations": [ + { + "predicateType": "https://slsa.dev/provenance/v1", + "bundle": { + "mediaType": "application/vnd.dev.sigstore.bundle.v0.3+json", + "dsseEnvelope": { + "payload": "{{payloadBase64}}", + "payloadType": "application/vnd.in-toto+json", + "signatures": [ + { + "sig": "MEUCIQC+fake+signature", + "keyid": "" + } + ] + }, + "verificationMaterial": { + "certificate": { + "rawBytes": "MIIFake..." + }, + "tlogEntries": [ + { + "logIndex": "12345", + "logId": { + "keyId": "fake-key-id" + }, + "kindVersion": { + "kind": "dsse", + "version": "0.0.1" + }, + "integratedTime": "1700000000", + "inclusionPromise": { + "signedEntryTimestamp": "MEUC..." + }, + "canonicalizedBody": "eyJ..." + } + ] + } + } + } + ] + } + """; + } + + private static string BuildProvenancePayload(string sourceRepository) + { + return $$""" + { + "_type": "https://in-toto.io/Statement/v1", + "subject": [ + { + "name": "pkg:npm/@playwright/cli@0.1.1", + "digest": { "sha512": "abc123" } + } + ], + "predicateType": "https://slsa.dev/provenance/v1", + "predicate": { + "buildDefinition": { + "buildType": "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + "externalParameters": { + "workflow": { + "ref": "refs/tags/v0.1.1", + "repository": "{{sourceRepository}}", + "path": ".github/workflows/publish.yml" + } + } + }, + "runDetails": { + "builder": { + "id": "https://github.com/actions/runner/github-hosted" + } + } + } + } + """; + } +} diff --git a/tests/Aspire.Cli.Tests/Aspire.Cli.Tests.csproj b/tests/Aspire.Cli.Tests/Aspire.Cli.Tests.csproj index 421fde25b5e..94c86d802ba 100644 --- a/tests/Aspire.Cli.Tests/Aspire.Cli.Tests.csproj +++ b/tests/Aspire.Cli.Tests/Aspire.Cli.Tests.csproj @@ -5,6 +5,7 @@ enable enable false + false false diff --git a/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs b/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs index b2ea398fc39..a9aaf74caf4 100644 --- a/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs +++ b/tests/Aspire.Cli.Tests/TestServices/FakePlaywrightServices.cs @@ -30,7 +30,7 @@ public Task InstallGlobalAsync(string tarballPath, CancellationToken cance /// internal sealed class FakeNpmProvenanceChecker : INpmProvenanceChecker { - public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, Func? validateWorkflowRef, CancellationToken cancellationToken) + public Task VerifyProvenanceAsync(string packageName, string version, string expectedSourceRepository, string expectedWorkflowPath, string expectedBuildType, Func? validateWorkflowRef, CancellationToken cancellationToken, string? sriIntegrity = null) => Task.FromResult(new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.Verified, From 95c68f8340d51afbbde38108280d9f5acee230e7 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 3 Mar 2026 14:26:22 +1100 Subject: [PATCH 14/15] Refactor SigstoreNpmProvenanceChecker for clarity Break the monolithic VerifyProvenanceAsync method into focused methods: - FetchAttestationJsonAsync: fetches attestation JSON from npm registry - ParseAttestation: parses JSON in a single pass, extracting both the Sigstore bundle and provenance data (eliminates duplicate JSON parsing) - ParseProvenanceFromStatement: extracts provenance fields from in-toto statement - VerifySigstoreBundleAsync: cryptographic Sigstore verification - VerifyProvenanceFields: field-level provenance checks (source repo, workflow, build type, workflow ref) Removes dependency on NpmProvenanceChecker.ParseProvenance() which was re-parsing the same JSON and iterating attestations a second time. Adds NpmAttestationParseResult type to carry both bundle and provenance data from a single parse pass. Adds comprehensive unit tests for ParseAttestation, ParseProvenanceFromStatement, and VerifyProvenanceFields covering success and failure cases. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Npm/SigstoreNpmProvenanceChecker.cs | 296 ++++++++++++------ .../SigstoreNpmProvenanceCheckerTests.cs | 184 ++++++++++- 2 files changed, 368 insertions(+), 112 deletions(-) diff --git a/src/Aspire.Cli/Npm/SigstoreNpmProvenanceChecker.cs b/src/Aspire.Cli/Npm/SigstoreNpmProvenanceChecker.cs index cad83094f1c..03f46bcb6cb 100644 --- a/src/Aspire.Cli/Npm/SigstoreNpmProvenanceChecker.cs +++ b/src/Aspire.Cli/Npm/SigstoreNpmProvenanceChecker.cs @@ -1,12 +1,35 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.Json; using System.Text.Json.Nodes; using Microsoft.Extensions.Logging; using Sigstore; namespace Aspire.Cli.Npm; +/// +/// The parsed result of an npm attestation response, containing both the Sigstore bundle +/// and the provenance data extracted from the DSSE envelope in a single pass. +/// +internal sealed class NpmAttestationParseResult +{ + /// + /// Gets the outcome of the parse operation. + /// + public required ProvenanceVerificationOutcome Outcome { get; init; } + + /// + /// Gets the raw Sigstore bundle JSON node for deserialization by the Sigstore library. + /// + public JsonNode? BundleNode { get; init; } + + /// + /// Gets the provenance data extracted from the DSSE envelope payload. + /// + public NpmProvenanceData? Provenance { get; init; } +} + /// /// Verifies npm package provenance by cryptographically verifying Sigstore bundles /// from the npm registry attestations API using the Sigstore .NET library. @@ -27,8 +50,37 @@ public async Task VerifyProvenanceAsync( CancellationToken cancellationToken, string? sriIntegrity = null) { - // Gate 1: Fetch attestations from the npm registry. - string json; + var json = await FetchAttestationJsonAsync(packageName, version, cancellationToken).ConfigureAwait(false); + if (json is null) + { + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationFetchFailed }; + } + + var attestation = ParseAttestation(json); + if (attestation.Outcome is not ProvenanceVerificationOutcome.Verified) + { + return new ProvenanceVerificationResult { Outcome = attestation.Outcome, Provenance = attestation.Provenance }; + } + + var sigstoreFailure = await VerifySigstoreBundleAsync( + attestation.BundleNode!, expectedSourceRepository, sriIntegrity, + packageName, version, cancellationToken).ConfigureAwait(false); + if (sigstoreFailure is not null) + { + return sigstoreFailure; + } + + return VerifyProvenanceFields( + attestation.Provenance!, expectedSourceRepository, expectedWorkflowPath, + expectedBuildType, validateWorkflowRef); + } + + /// + /// Fetches the attestation JSON from the npm registry for the given package and version. + /// + private async Task FetchAttestationJsonAsync( + string packageName, string version, CancellationToken cancellationToken) + { try { var encodedPackage = Uri.EscapeDataString(packageName); @@ -40,35 +92,143 @@ public async Task VerifyProvenanceAsync( if (!response.IsSuccessStatusCode) { logger.LogDebug("Failed to fetch attestations: HTTP {StatusCode}", response.StatusCode); - return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationFetchFailed }; + return null; } - json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + return await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); } catch (HttpRequestException ex) { logger.LogDebug(ex, "Failed to fetch attestations for {Package}@{Version}", packageName, version); - return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationFetchFailed }; + return null; } + } - // Gate 2: Find the SLSA provenance attestation and extract its Sigstore bundle. - JsonNode? bundleNode; + /// + /// Parses the npm attestation JSON in a single pass, extracting both the Sigstore bundle + /// node and the provenance data from the SLSA provenance attestation's DSSE envelope. + /// + internal static NpmAttestationParseResult ParseAttestation(string attestationJson) + { + JsonNode? doc; try { - bundleNode = FindSlsaProvenanceBundle(json); + doc = JsonNode.Parse(attestationJson); + } + catch (JsonException) + { + return new NpmAttestationParseResult { Outcome = ProvenanceVerificationOutcome.AttestationParseFailed }; + } + + var attestations = doc?["attestations"]?.AsArray(); + if (attestations is null || attestations.Count == 0) + { + return new NpmAttestationParseResult { Outcome = ProvenanceVerificationOutcome.SlsaProvenanceNotFound }; + } + + foreach (var attestation in attestations) + { + var predicateType = attestation?["predicateType"]?.GetValue(); + if (!string.Equals(predicateType, SlsaProvenancePredicateType, StringComparison.Ordinal)) + { + continue; + } + + var bundleNode = attestation?["bundle"]; if (bundleNode is null) { - return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.SlsaProvenanceNotFound }; + return new NpmAttestationParseResult { Outcome = ProvenanceVerificationOutcome.SlsaProvenanceNotFound }; + } + + var payload = bundleNode["dsseEnvelope"]?["payload"]?.GetValue(); + if (payload is null) + { + return new NpmAttestationParseResult + { + Outcome = ProvenanceVerificationOutcome.PayloadDecodeFailed, + BundleNode = bundleNode + }; + } + + byte[] decodedBytes; + try + { + decodedBytes = Convert.FromBase64String(payload); + } + catch (FormatException) + { + return new NpmAttestationParseResult + { + Outcome = ProvenanceVerificationOutcome.PayloadDecodeFailed, + BundleNode = bundleNode + }; + } + + var provenance = ParseProvenanceFromStatement(decodedBytes); + if (provenance is null) + { + return new NpmAttestationParseResult + { + Outcome = ProvenanceVerificationOutcome.AttestationParseFailed, + BundleNode = bundleNode + }; } + + var outcome = provenance.SourceRepository is null + ? ProvenanceVerificationOutcome.SourceRepositoryNotFound + : ProvenanceVerificationOutcome.Verified; + + return new NpmAttestationParseResult + { + Outcome = outcome, + BundleNode = bundleNode, + Provenance = provenance + }; + } + + return new NpmAttestationParseResult { Outcome = ProvenanceVerificationOutcome.SlsaProvenanceNotFound }; + } + + /// + /// Extracts provenance fields from a decoded in-toto statement. + /// + internal static NpmProvenanceData? ParseProvenanceFromStatement(byte[] statementBytes) + { + try + { + var statement = JsonNode.Parse(statementBytes); + var predicate = statement?["predicate"]; + var buildDefinition = predicate?["buildDefinition"]; + var workflow = buildDefinition?["externalParameters"]?["workflow"]; + + return new NpmProvenanceData + { + SourceRepository = workflow?["repository"]?.GetValue(), + WorkflowPath = workflow?["path"]?.GetValue(), + WorkflowRef = workflow?["ref"]?.GetValue(), + BuilderId = predicate?["runDetails"]?["builder"]?["id"]?.GetValue(), + BuildType = buildDefinition?["buildType"]?.GetValue() + }; } - catch (Exception ex) when (ex is System.Text.Json.JsonException or InvalidOperationException) + catch (JsonException) { - logger.LogDebug(ex, "Failed to parse attestation response for {Package}@{Version}", packageName, version); - return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationParseFailed }; + return null; } + } - // Gate 3: Cryptographically verify the Sigstore bundle using the Sigstore library. - // This verifies the Fulcio certificate chain, Rekor transparency log inclusion, and OIDC identity. + /// + /// Cryptographically verifies the Sigstore bundle using the Sigstore library. + /// Checks the Fulcio certificate chain, Rekor transparency log inclusion, and OIDC identity. + /// + /// null if verification succeeded; otherwise a failure result. + private async Task VerifySigstoreBundleAsync( + JsonNode bundleNode, + string expectedSourceRepository, + string? sriIntegrity, + string packageName, + string version, + CancellationToken cancellationToken) + { var bundleJson = bundleNode.ToJsonString(); SigstoreBundle bundle; try @@ -81,12 +241,10 @@ public async Task VerifyProvenanceAsync( return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationParseFailed }; } - // Extract the owner and repo from the expected source repository URL. - // Expected format: "https://github.com/{owner}/{repo}" if (!TryParseGitHubOwnerRepo(expectedSourceRepository, out var owner, out var repo)) { logger.LogWarning("Could not parse GitHub owner/repo from expected source repository: {ExpectedSourceRepository}", expectedSourceRepository); - return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.SourceRepositoryNotFound }; + return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.SourceRepositoryMismatch }; } var verifier = new SigstoreVerifier(); @@ -102,27 +260,24 @@ public async Task VerifyProvenanceAsync( if (sriIntegrity is not null && sriIntegrity.StartsWith("sha512-", StringComparison.OrdinalIgnoreCase)) { - // Verify the bundle against the tarball's SHA-512 digest from the SRI integrity string. var hashBase64 = sriIntegrity["sha512-".Length..]; var digestBytes = Convert.FromBase64String(hashBase64); (success, result) = await verifier.TryVerifyDigestAsync( - digestBytes, HashAlgorithmType.Sha512, bundle, policy).ConfigureAwait(false); + digestBytes, HashAlgorithmType.Sha512, bundle, policy, cancellationToken).ConfigureAwait(false); } else { - // No integrity hash available — verify using the DSSE envelope payload bytes. - // The DSSE payload is the in-toto statement that was signed. - var payloadNode = bundleNode["dsseEnvelope"]?["payload"]?.GetValue(); - if (payloadNode is null) + var payloadBase64 = bundleNode["dsseEnvelope"]?["payload"]?.GetValue(); + if (payloadBase64 is null) { logger.LogDebug("No DSSE payload found in bundle for {Package}@{Version}", packageName, version); return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.PayloadDecodeFailed }; } - var payloadBytes = Convert.FromBase64String(payloadNode); + var payloadBytes = Convert.FromBase64String(payloadBase64); (success, result) = await verifier.TryVerifyAsync( - payloadBytes, bundle, policy).ConfigureAwait(false); + payloadBytes, bundle, policy, cancellationToken).ConfigureAwait(false); } if (!success) @@ -136,48 +291,29 @@ public async Task VerifyProvenanceAsync( logger.LogDebug( "Sigstore verification passed for {Package}@{Version}. Signed by: {Signer}", packageName, version, result?.SignerIdentity?.SubjectAlternativeName); + + return null; } catch (Exception ex) { logger.LogWarning(ex, "Sigstore verification threw an exception for {Package}@{Version}", packageName, version); return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationParseFailed }; } + } - // Gate 4: Parse the DSSE envelope payload for provenance data and apply field-level checks. - NpmProvenanceData provenance; - try - { - var parseResult = NpmProvenanceChecker.ParseProvenance(json); - if (parseResult is null) - { - return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.SlsaProvenanceNotFound }; - } - - provenance = parseResult.Value.Provenance; - if (parseResult.Value.Outcome is not ProvenanceVerificationOutcome.Verified) - { - return new ProvenanceVerificationResult - { - Outcome = parseResult.Value.Outcome, - Provenance = provenance - }; - } - } - catch (System.Text.Json.JsonException ex) - { - logger.LogDebug(ex, "Failed to parse provenance data from attestation for {Package}@{Version}", packageName, version); - return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.AttestationParseFailed }; - } - - logger.LogDebug("SLSA provenance source repository: {SourceRepository}", provenance.SourceRepository); - - // Gate 5: Verify the source repository matches. + /// + /// Verifies that the extracted provenance fields match the expected values. + /// Checks source repository, workflow path, build type, and workflow ref in order. + /// + internal static ProvenanceVerificationResult VerifyProvenanceFields( + NpmProvenanceData provenance, + string expectedSourceRepository, + string expectedWorkflowPath, + string expectedBuildType, + Func? validateWorkflowRef) + { if (!string.Equals(provenance.SourceRepository, expectedSourceRepository, StringComparison.OrdinalIgnoreCase)) { - logger.LogWarning( - "Provenance verification failed: expected source repository {Expected} but attestation says {Actual}", - expectedSourceRepository, provenance.SourceRepository); - return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.SourceRepositoryMismatch, @@ -185,13 +321,8 @@ public async Task VerifyProvenanceAsync( }; } - // Gate 6: Verify the workflow path matches. if (!string.Equals(provenance.WorkflowPath, expectedWorkflowPath, StringComparison.Ordinal)) { - logger.LogWarning( - "Provenance verification failed: expected workflow path {Expected} but attestation says {Actual}", - expectedWorkflowPath, provenance.WorkflowPath); - return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.WorkflowMismatch, @@ -199,13 +330,8 @@ public async Task VerifyProvenanceAsync( }; } - // Gate 7: Verify the build type matches. if (!string.Equals(provenance.BuildType, expectedBuildType, StringComparison.Ordinal)) { - logger.LogWarning( - "Provenance verification failed: expected build type {Expected} but attestation says {Actual}", - expectedBuildType, provenance.BuildType); - return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.BuildTypeMismatch, @@ -213,15 +339,10 @@ public async Task VerifyProvenanceAsync( }; } - // Gate 8: Verify the workflow ref using the caller-provided validation callback. if (validateWorkflowRef is not null) { if (!WorkflowRefInfo.TryParse(provenance.WorkflowRef, out var refInfo) || refInfo is null) { - logger.LogWarning( - "Provenance verification failed: could not parse workflow ref {WorkflowRef}", - provenance.WorkflowRef); - return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.WorkflowRefMismatch, @@ -231,10 +352,6 @@ public async Task VerifyProvenanceAsync( if (!validateWorkflowRef(refInfo)) { - logger.LogWarning( - "Provenance verification failed: workflow ref {WorkflowRef} did not pass validation", - provenance.WorkflowRef); - return new ProvenanceVerificationResult { Outcome = ProvenanceVerificationOutcome.WorkflowRefMismatch, @@ -250,31 +367,6 @@ public async Task VerifyProvenanceAsync( }; } - /// - /// Finds the Sigstore bundle JSON node for the SLSA provenance attestation. - /// - internal static JsonNode? FindSlsaProvenanceBundle(string attestationJson) - { - var doc = JsonNode.Parse(attestationJson); - var attestations = doc?["attestations"]?.AsArray(); - - if (attestations is null || attestations.Count == 0) - { - return null; - } - - foreach (var attestation in attestations) - { - var predicateType = attestation?["predicateType"]?.GetValue(); - if (string.Equals(predicateType, SlsaProvenancePredicateType, StringComparison.Ordinal)) - { - return attestation?["bundle"]; - } - } - - return null; - } - /// /// Parses a GitHub repository URL into owner and repo components. /// diff --git a/tests/Aspire.Cli.Tests/Agents/SigstoreNpmProvenanceCheckerTests.cs b/tests/Aspire.Cli.Tests/Agents/SigstoreNpmProvenanceCheckerTests.cs index b4cc08f4e21..c0a6541e9fb 100644 --- a/tests/Aspire.Cli.Tests/Agents/SigstoreNpmProvenanceCheckerTests.cs +++ b/tests/Aspire.Cli.Tests/Agents/SigstoreNpmProvenanceCheckerTests.cs @@ -8,18 +8,23 @@ namespace Aspire.Cli.Tests.Agents; public class SigstoreNpmProvenanceCheckerTests { [Fact] - public void FindSlsaProvenanceBundle_WithValidSlsaAttestation_ReturnsBundle() + public void ParseAttestation_WithValidSlsaAttestation_ReturnsBundleAndProvenance() { var json = BuildAttestationJsonWithBundle("https://github.com/microsoft/playwright-cli"); - var bundle = SigstoreNpmProvenanceChecker.FindSlsaProvenanceBundle(json); + var result = SigstoreNpmProvenanceChecker.ParseAttestation(json); - Assert.NotNull(bundle); - Assert.NotNull(bundle["dsseEnvelope"]); + Assert.Equal(ProvenanceVerificationOutcome.Verified, result.Outcome); + Assert.NotNull(result.BundleNode); + Assert.NotNull(result.BundleNode["dsseEnvelope"]); + Assert.NotNull(result.Provenance); + Assert.Equal("https://github.com/microsoft/playwright-cli", result.Provenance.SourceRepository); + Assert.Equal(".github/workflows/publish.yml", result.Provenance.WorkflowPath); + Assert.Equal("refs/tags/v0.1.1", result.Provenance.WorkflowRef); } [Fact] - public void FindSlsaProvenanceBundle_WithNoSlsaPredicate_ReturnsNull() + public void ParseAttestation_WithNoSlsaPredicate_ReturnsSlsaProvenanceNotFound() { var json = """ { @@ -36,19 +41,178 @@ public void FindSlsaProvenanceBundle_WithNoSlsaPredicate_ReturnsNull() } """; - var bundle = SigstoreNpmProvenanceChecker.FindSlsaProvenanceBundle(json); + var result = SigstoreNpmProvenanceChecker.ParseAttestation(json); - Assert.Null(bundle); + Assert.Equal(ProvenanceVerificationOutcome.SlsaProvenanceNotFound, result.Outcome); } [Fact] - public void FindSlsaProvenanceBundle_WithEmptyAttestations_ReturnsNull() + public void ParseAttestation_WithEmptyAttestations_ReturnsSlsaProvenanceNotFound() { var json = """{"attestations": []}"""; - var bundle = SigstoreNpmProvenanceChecker.FindSlsaProvenanceBundle(json); + var result = SigstoreNpmProvenanceChecker.ParseAttestation(json); - Assert.Null(bundle); + Assert.Equal(ProvenanceVerificationOutcome.SlsaProvenanceNotFound, result.Outcome); + } + + [Fact] + public void ParseAttestation_WithInvalidJson_ReturnsAttestationParseFailed() + { + var result = SigstoreNpmProvenanceChecker.ParseAttestation("not valid json {{{"); + + Assert.Equal(ProvenanceVerificationOutcome.AttestationParseFailed, result.Outcome); + } + + [Fact] + public void ParseAttestation_WithMissingPayload_ReturnsPayloadDecodeFailed() + { + var json = """ + { + "attestations": [ + { + "predicateType": "https://slsa.dev/provenance/v1", + "bundle": { + "dsseEnvelope": {} + } + } + ] + } + """; + + var result = SigstoreNpmProvenanceChecker.ParseAttestation(json); + + Assert.Equal(ProvenanceVerificationOutcome.PayloadDecodeFailed, result.Outcome); + Assert.NotNull(result.BundleNode); + } + + [Fact] + public void ParseProvenanceFromStatement_WithValidStatement_ReturnsProvenance() + { + var payload = BuildProvenancePayload("https://github.com/microsoft/playwright-cli"); + var bytes = System.Text.Encoding.UTF8.GetBytes(payload); + + var provenance = SigstoreNpmProvenanceChecker.ParseProvenanceFromStatement(bytes); + + Assert.NotNull(provenance); + Assert.Equal("https://github.com/microsoft/playwright-cli", provenance.SourceRepository); + Assert.Equal(".github/workflows/publish.yml", provenance.WorkflowPath); + Assert.Equal("refs/tags/v0.1.1", provenance.WorkflowRef); + Assert.Equal("https://github.com/actions/runner/github-hosted", provenance.BuilderId); + Assert.Equal("https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", provenance.BuildType); + } + + [Fact] + public void ParseProvenanceFromStatement_WithInvalidJson_ReturnsNull() + { + var bytes = System.Text.Encoding.UTF8.GetBytes("not json"); + + var provenance = SigstoreNpmProvenanceChecker.ParseProvenanceFromStatement(bytes); + + Assert.Null(provenance); + } + + [Fact] + public void VerifyProvenanceFields_WithAllFieldsMatching_ReturnsVerified() + { + var provenance = new NpmProvenanceData + { + SourceRepository = "https://github.com/microsoft/playwright-cli", + WorkflowPath = ".github/workflows/publish.yml", + BuildType = "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + WorkflowRef = "refs/tags/v0.1.1", + BuilderId = "https://github.com/actions/runner/github-hosted" + }; + + var result = SigstoreNpmProvenanceChecker.VerifyProvenanceFields( + provenance, + "https://github.com/microsoft/playwright-cli", + ".github/workflows/publish.yml", + "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + refInfo => refInfo.Kind == "tags"); + + Assert.Equal(ProvenanceVerificationOutcome.Verified, result.Outcome); + } + + [Fact] + public void VerifyProvenanceFields_WithSourceRepoMismatch_ReturnsSourceRepositoryMismatch() + { + var provenance = new NpmProvenanceData + { + SourceRepository = "https://github.com/evil/repo", + WorkflowPath = ".github/workflows/publish.yml", + BuildType = "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + }; + + var result = SigstoreNpmProvenanceChecker.VerifyProvenanceFields( + provenance, + "https://github.com/microsoft/playwright-cli", + ".github/workflows/publish.yml", + "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + null); + + Assert.Equal(ProvenanceVerificationOutcome.SourceRepositoryMismatch, result.Outcome); + } + + [Fact] + public void VerifyProvenanceFields_WithWorkflowMismatch_ReturnsWorkflowMismatch() + { + var provenance = new NpmProvenanceData + { + SourceRepository = "https://github.com/microsoft/playwright-cli", + WorkflowPath = ".github/workflows/evil.yml", + BuildType = "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + }; + + var result = SigstoreNpmProvenanceChecker.VerifyProvenanceFields( + provenance, + "https://github.com/microsoft/playwright-cli", + ".github/workflows/publish.yml", + "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + null); + + Assert.Equal(ProvenanceVerificationOutcome.WorkflowMismatch, result.Outcome); + } + + [Fact] + public void VerifyProvenanceFields_WithBuildTypeMismatch_ReturnsBuildTypeMismatch() + { + var provenance = new NpmProvenanceData + { + SourceRepository = "https://github.com/microsoft/playwright-cli", + WorkflowPath = ".github/workflows/publish.yml", + BuildType = "https://evil.example.com/build/v1", + }; + + var result = SigstoreNpmProvenanceChecker.VerifyProvenanceFields( + provenance, + "https://github.com/microsoft/playwright-cli", + ".github/workflows/publish.yml", + "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + null); + + Assert.Equal(ProvenanceVerificationOutcome.BuildTypeMismatch, result.Outcome); + } + + [Fact] + public void VerifyProvenanceFields_WithWorkflowRefValidationFailure_ReturnsWorkflowRefMismatch() + { + var provenance = new NpmProvenanceData + { + SourceRepository = "https://github.com/microsoft/playwright-cli", + WorkflowPath = ".github/workflows/publish.yml", + BuildType = "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + WorkflowRef = "refs/heads/main" + }; + + var result = SigstoreNpmProvenanceChecker.VerifyProvenanceFields( + provenance, + "https://github.com/microsoft/playwright-cli", + ".github/workflows/publish.yml", + "https://slsa-framework.github.io/github-actions-buildtypes/workflow/v1", + refInfo => refInfo.Kind == "tags"); + + Assert.Equal(ProvenanceVerificationOutcome.WorkflowRefMismatch, result.Outcome); } [Theory] From 31157ef448e1088de8f6e80f5c7ebae1ce1fd461 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 3 Mar 2026 14:52:40 +1100 Subject: [PATCH 15/15] Add libsodium to nuget-org package source mapping libsodium is a transitive dependency of NSec.Cryptography (used by Sigstore) and needs to be mapped to the nuget-org source for CI restore to succeed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- NuGet.config | 1 + 1 file changed, 1 insertion(+) diff --git a/NuGet.config b/NuGet.config index c923efacf56..6ffc9600bba 100644 --- a/NuGet.config +++ b/NuGet.config @@ -50,6 +50,7 @@ +