diff --git a/.cspell.json b/.cspell.json index c4f0059..9bd05e0 100644 --- a/.cspell.json +++ b/.cspell.json @@ -25,6 +25,7 @@ "opencover", "reqstream", "snupkg", + "tracematrix", "trx", "yamllint" ], diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ebafde7..c201828 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -234,6 +234,21 @@ jobs: buildmark --help || { echo "✗ Help command failed"; exit 1; } echo "✓ Help command succeeded" + - name: Run BuildMark self-validation + shell: bash + run: | + echo "Running BuildMark self-validation..." + buildmark --validate --results validation-${{ matrix.os }}-dotnet${{ matrix.dotnet-version }}.trx \ + || { echo "✗ Self-validation failed"; exit 1; } + echo "✓ Self-validation succeeded" + + - name: Upload validation test results + if: always() + uses: actions/upload-artifact@v6 + with: + name: validation-test-results-${{ matrix.os }}-dotnet${{ matrix.dotnet-version }} + path: validation-${{ matrix.os }}-dotnet${{ matrix.dotnet-version }}.trx + build-docs: name: Build Documents runs-on: windows-latest @@ -278,12 +293,12 @@ jobs: - name: Install npm dependencies run: npm install - - name: Download All Test Results + - name: Download all test results uses: actions/download-artifact@v7 with: - pattern: test-results-* path: test-results - merge-multiple: true + pattern: '*test-results*' + continue-on-error: true - name: Download CodeQL SARIF uses: actions/download-artifact@v7 @@ -291,6 +306,15 @@ jobs: name: codeql-sarif path: codeql-results + - name: Generate Requirements Report and Trace Matrix + run: > + dotnet reqstream + --requirements requirements.yaml + --tests "test-results/**/*.trx" + --report docs/requirements/requirements.md + --matrix docs/tracematrix/tracematrix.md + --enforce + - name: Generate CodeQL Quality Report with SarifMark run: > dotnet sarifmark @@ -355,6 +379,38 @@ jobs: docs/buildnotes/buildnotes.html "docs/BuildMark Build Notes.pdf" + - name: Generate Requirements HTML with Pandoc + shell: bash + run: > + dotnet pandoc + --defaults docs/requirements/definition.yaml + --metadata version="${{ inputs.version }}" + --metadata date="$(date +'%Y-%m-%d')" + --filter node_modules/.bin/mermaid-filter.cmd + --output docs/requirements/requirements.html + + - name: Generate Requirements PDF with Weasyprint + run: > + dotnet weasyprint + docs/requirements/requirements.html + "docs/BuildMark Requirements.pdf" + + - name: Generate Trace Matrix HTML with Pandoc + shell: bash + run: > + dotnet pandoc + --defaults docs/tracematrix/definition.yaml + --metadata version="${{ inputs.version }}" + --metadata date="$(date +'%Y-%m-%d')" + --filter node_modules/.bin/mermaid-filter.cmd + --output docs/tracematrix/tracematrix.html + + - name: Generate Trace Matrix PDF with Weasyprint + run: > + dotnet weasyprint + docs/tracematrix/tracematrix.html + "docs/BuildMark Trace Matrix.pdf" + - name: Generate Code Quality HTML with Pandoc shell: bash run: > diff --git a/docs/requirements/definition.yaml b/docs/requirements/definition.yaml new file mode 100644 index 0000000..a0f3371 --- /dev/null +++ b/docs/requirements/definition.yaml @@ -0,0 +1,11 @@ +--- +resource-path: + - docs/requirements + - docs/template +input-files: + - docs/requirements/title.txt + - docs/requirements/introduction.md + - docs/requirements/requirements.md +template: template.html +table-of-contents: true +number-sections: true diff --git a/docs/requirements/introduction.md b/docs/requirements/introduction.md new file mode 100644 index 0000000..4275dc0 --- /dev/null +++ b/docs/requirements/introduction.md @@ -0,0 +1,31 @@ +# Introduction + +This document contains the requirements for the BuildMark project. + +## Purpose + +BuildMark is a .NET command-line tool that generates comprehensive markdown build notes reports from +Git repository history and GitHub issues. It analyzes commits, pull requests, and issues to create +human-readable build notes, making it easy to integrate release documentation into CI/CD pipelines +and documentation workflows. + +## Scope + +This requirements document covers: + +- Command-line interface and options +- Git repository integration +- GitHub integration for issues and pull requests +- Markdown report generation capabilities +- Validation and self-testing features +- Data models and repository connectors +- Platform support for Windows, Linux, and multiple .NET runtimes + +## Audience + +This document is intended for: + +- Software developers working on BuildMark +- Quality assurance teams validating requirements +- Project stakeholders reviewing project scope +- Users understanding the tool's capabilities diff --git a/docs/requirements/title.txt b/docs/requirements/title.txt new file mode 100644 index 0000000..db6608c --- /dev/null +++ b/docs/requirements/title.txt @@ -0,0 +1,17 @@ +--- +title: BuildMark Requirements +subtitle: Requirements Specification for the BuildMark Tool +author: DEMA Consulting +description: Requirements Specification for the BuildMark Tool +lang: en-US +keywords: + - BuildMark + - Requirements + - Specification + - .NET + - YAML + - Command-Line Tool + - Git + - GitHub + - Markdown +--- diff --git a/docs/tracematrix/definition.yaml b/docs/tracematrix/definition.yaml new file mode 100644 index 0000000..ba93d57 --- /dev/null +++ b/docs/tracematrix/definition.yaml @@ -0,0 +1,11 @@ +--- +resource-path: + - docs/tracematrix + - docs/template +input-files: + - docs/tracematrix/title.txt + - docs/tracematrix/introduction.md + - docs/tracematrix/tracematrix.md +template: template.html +table-of-contents: true +number-sections: true diff --git a/docs/tracematrix/introduction.md b/docs/tracematrix/introduction.md new file mode 100644 index 0000000..83adda5 --- /dev/null +++ b/docs/tracematrix/introduction.md @@ -0,0 +1,48 @@ +# Introduction + +This document contains the requirements trace matrix for the BuildMark project. + +## Purpose + +The trace matrix links requirements to their corresponding test cases, ensuring complete +test coverage and traceability from requirements to implementation. + +## Test Sources + +Requirements traceability in BuildMark uses two types of tests: + +- **Unit and Integration Tests**: Standard MSTest tests that verify code functionality +- **Self-Validation Tests**: Built-in validation tests run via `buildmark --validate --results` + +To generate complete traceability reports, both test result files must be included: + +```bash +# Run unit and integration tests +dotnet test --configuration Release --results-directory test-results --logger "trx" + +# Run validation tests +buildmark --validate --results validation.trx + +# Verify requirements traceability +dotnet reqstream --requirements requirements.yaml \ + --tests "test-results/**/*.trx" \ + --tests validation.trx \ + --enforce +``` + +## Interpretation + +The trace matrix shows: + +- **Requirement ID**: Unique identifier for each requirement +- **Requirement Title**: Brief description of the requirement +- **Test Coverage**: List of test cases that verify the requirement +- **Status**: Indication of whether all mapped tests pass + +## Coverage Requirements + +All requirements must have: + +- At least one test case mapped to verify the requirement +- All mapped tests must pass for the requirement to be satisfied +- Tests must execute on supported platforms and .NET runtimes diff --git a/docs/tracematrix/title.txt b/docs/tracematrix/title.txt new file mode 100644 index 0000000..4708df9 --- /dev/null +++ b/docs/tracematrix/title.txt @@ -0,0 +1,14 @@ +--- +title: BuildMark Trace Matrix +subtitle: Test Traceability Matrix for the BuildMark Tool +author: DEMA Consulting +description: Test Traceability Matrix for the BuildMark Tool +lang: en-US +keywords: + - BuildMark + - Trace Matrix + - Traceability + - Testing + - .NET + - Command-Line Tool +--- diff --git a/requirements.yaml b/requirements.yaml new file mode 100644 index 0000000..4fbc35e --- /dev/null +++ b/requirements.yaml @@ -0,0 +1,216 @@ +--- +sections: + - title: "BuildMark Requirements" + sections: + - title: "Command-Line Interface" + requirements: + - id: "CLI-001" + title: "The tool shall provide a command-line interface." + tests: + - "IntegrationTest_VersionFlag_OutputsVersion" + - "IntegrationTest_HelpFlag_OutputsUsageInformation" + + - id: "CLI-002" + title: "The tool shall display version information when requested." + tests: + - "IntegrationTest_VersionFlag_OutputsVersion" + + - id: "CLI-003" + title: "The tool shall display help information when requested." + tests: + - "IntegrationTest_HelpFlag_OutputsUsageInformation" + + - id: "CLI-004" + title: "The tool shall support silent mode to suppress console output." + tests: + - "IntegrationTest_SilentFlag_SuppressesOutput" + + - id: "CLI-005" + title: "The tool shall support writing output to a log file." + tests: + - "IntegrationTest_LogParameter_IsAccepted" + + - id: "CLI-006" + title: "The tool shall support specifying a build version." + tests: + - "IntegrationTest_BuildVersionParameter_IsAccepted" + + - id: "CLI-007" + title: "The tool shall validate command-line arguments." + tests: + - "IntegrationTest_InvalidArgument_ShowsError" + - "Context_Create_UnsupportedArgument_ThrowsArgumentException" + - "Context_Create_BuildVersionWithoutValue_ThrowsArgumentException" + - "Context_Create_ReportWithoutValue_ThrowsArgumentException" + - "Context_Create_ReportDepthWithoutValue_ThrowsArgumentException" + - "Context_Create_ReportDepthWithNonIntegerValue_ThrowsArgumentException" + - "Context_Create_ReportDepthWithZeroValue_ThrowsArgumentException" + - "Context_Create_ReportDepthWithNegativeValue_ThrowsArgumentException" + - "Context_Create_ResultsWithoutValue_ThrowsArgumentException" + - "Context_Create_LogWithoutValue_ThrowsArgumentException" + + - id: "CLI-008" + title: "The tool shall handle invalid file paths gracefully." + tests: + - "Context_Create_InvalidLogFilePath_ThrowsInvalidOperationException" + + - id: "CLI-009" + title: "The tool shall return appropriate exit codes." + tests: + - "Context_ExitCode_NoErrors_RemainsZero" + - "Context_WriteError_SetsExitCodeToOne" + + - title: "GitHub Integration" + requirements: + - id: "GH-001" + title: "The tool shall generate build notes from GitHub repositories." + tests: + - "BuildMark_GitIntegration" + - "BuildMark_MarkdownReportGeneration" + + - id: "GH-002" + title: "The tool shall track issues and pull requests from GitHub." + tests: + - "BuildMark_IssueTracking" + + - id: "GH-003" + title: "The tool shall identify changes between version tags." + tests: + - "BuildMark_GitIntegration" + + - title: "Report Generation" + requirements: + - id: "RPT-001" + title: "The tool shall generate markdown reports." + tests: + - "BuildMark_MarkdownReportGeneration" + - "IntegrationTest_ReportParameter_IsAccepted" + + - id: "RPT-002" + title: "The tool shall support configurable report depth." + tests: + - "IntegrationTest_ReportDepthParameter_IsAccepted" + - "BuildInformation_ToMarkdown_RespectsCustomHeadingDepth" + + - id: "RPT-003" + title: "The tool shall report version information in build notes." + tests: + - "BuildMark_MarkdownReportGeneration" + + - id: "RPT-004" + title: "The tool shall report changes in build notes." + tests: + - "BuildMark_MarkdownReportGeneration" + - "BuildInformation_ToMarkdown_DisplaysNAForEmptyChanges" + + - id: "RPT-005" + title: "The tool shall report bug fixes in build notes." + tests: + - "BuildMark_MarkdownReportGeneration" + - "BuildInformation_ToMarkdown_DisplaysNAForEmptyBugs" + + - id: "RPT-006" + title: "The tool shall support including known issues in build notes." + tests: + - "BuildMark_KnownIssuesReporting" + + - id: "RPT-007" + title: "The tool shall support filtering build notes by version range." + tests: + - "BuildMark_MarkdownReportGeneration" + + - id: "RPT-008" + title: "The tool shall include hyperlinks to issues and pull requests." + tests: + - "BuildMark_MarkdownReportGeneration" + - "BuildInformation_ToMarkdown_IncludesIssueLinks" + - "BuildInformation_ToMarkdown_IncludesFullChangelogWhenLinkPresent" + + - id: "RPT-009" + title: "The tool shall format build notes with proper markdown structure." + tests: + - "BuildMark_MarkdownReportGeneration" + - "BuildInformation_ToMarkdown_RespectsCustomHeadingDepth" + - "BuildInformation_ToMarkdown_UsesCorrectTableWidths" + - "BuildInformation_ToMarkdown_HandlesFirstReleaseWithNA" + - "BuildInformation_ToMarkdown_ExcludesFullChangelogWhenNoBaseline" + + - title: "Validation and Testing" + requirements: + - id: "VAL-001" + title: "The tool shall support self-validation mode." + tests: + - "IntegrationTest_ValidateFlag_RunsSelfValidation" + + - id: "VAL-002" + title: "The tool shall write validation results to test result files." + tests: + - "IntegrationTest_ResultsParameter_IsAccepted" + + - id: "VAL-003" + title: "The tool shall support TRX format for test results." + tests: + - "IntegrationTest_ResultsParameter_IsAccepted" + + - id: "VAL-004" + title: "The tool shall support JUnit format for test results." + tests: + - "IntegrationTest_ResultsParameter_IsAccepted" + + - title: "Platform Support" + requirements: + - id: "PLT-001" + title: "The tool shall run on Windows operating systems." + # Test source pattern "windows@" ensures these tests ran on Windows. + # This filtering is necessary to prove Windows OS functionality. + tests: + - "windows@IntegrationTest_VersionFlag_OutputsVersion" + - "windows@IntegrationTest_HelpFlag_OutputsUsageInformation" + - "windows@IntegrationTest_ReportParameter_IsAccepted" + - "windows@BuildMark_MarkdownReportGeneration" + - "windows@BuildMark_GitIntegration" + - "windows@BuildMark_IssueTracking" + - "windows@BuildMark_KnownIssuesReporting" + + - id: "PLT-002" + title: "The tool shall run on Linux operating systems." + # Test source pattern "ubuntu@" ensures these tests ran on Linux. + # This filtering is necessary to prove Linux OS functionality. + tests: + - "ubuntu@IntegrationTest_VersionFlag_OutputsVersion" + - "ubuntu@IntegrationTest_HelpFlag_OutputsUsageInformation" + - "ubuntu@IntegrationTest_ReportParameter_IsAccepted" + - "ubuntu@BuildMark_MarkdownReportGeneration" + - "ubuntu@BuildMark_GitIntegration" + - "ubuntu@BuildMark_IssueTracking" + - "ubuntu@BuildMark_KnownIssuesReporting" + + - id: "PLT-003" + title: "The tool shall support .NET 8.0 runtime." + # Test source pattern "dotnet8.x@" ensures these tests ran on .NET 8.0. + # This filtering is necessary to prove .NET 8.0 runtime support. + tests: + - "dotnet8.x@BuildMark_MarkdownReportGeneration" + - "dotnet8.x@BuildMark_GitIntegration" + - "dotnet8.x@BuildMark_IssueTracking" + - "dotnet8.x@BuildMark_KnownIssuesReporting" + + - id: "PLT-004" + title: "The tool shall support .NET 9.0 runtime." + # Test source pattern "dotnet9.x@" ensures these tests ran on .NET 9.0. + # This filtering is necessary to prove .NET 9.0 runtime support. + tests: + - "dotnet9.x@BuildMark_MarkdownReportGeneration" + - "dotnet9.x@BuildMark_GitIntegration" + - "dotnet9.x@BuildMark_IssueTracking" + - "dotnet9.x@BuildMark_KnownIssuesReporting" + + - id: "PLT-005" + title: "The tool shall support .NET 10.0 runtime." + # Test source pattern "dotnet10.x@" ensures these tests ran on .NET 10.0. + # This filtering is necessary to prove .NET 10.0 runtime support. + tests: + - "dotnet10.x@BuildMark_MarkdownReportGeneration" + - "dotnet10.x@BuildMark_GitIntegration" + - "dotnet10.x@BuildMark_IssueTracking" + - "dotnet10.x@BuildMark_KnownIssuesReporting" diff --git a/src/DemaConsulting.BuildMark/Context.cs b/src/DemaConsulting.BuildMark/Context.cs index da4e52b..e0dfb31 100644 --- a/src/DemaConsulting.BuildMark/Context.cs +++ b/src/DemaConsulting.BuildMark/Context.cs @@ -85,6 +85,11 @@ internal sealed class Context : IDisposable /// public int ExitCode => _hasErrors ? 1 : 0; + /// + /// Gets the repository connector factory for creating connectors. + /// + public Func? ConnectorFactory { get; private init; } + /// /// Private constructor - use Create factory method instead. /// @@ -99,6 +104,18 @@ private Context() /// A new Context instance. /// Thrown when arguments are invalid. public static Context Create(string[] args) + { + return Create(args, null); + } + + /// + /// Creates a Context instance from command-line arguments with optional connector factory. + /// + /// Command-line arguments. + /// Optional repository connector factory for testing. + /// A new Context instance. + /// Thrown when arguments are invalid. + public static Context Create(string[] args, Func? connectorFactory) { var parser = new ArgumentParser(); parser.ParseArguments(args); @@ -113,7 +130,8 @@ public static Context Create(string[] args) ReportFile = parser.ReportFile, ReportDepth = parser.ReportDepth, IncludeKnownIssues = parser.IncludeKnownIssues, - ResultsFile = parser.ResultsFile + ResultsFile = parser.ResultsFile, + ConnectorFactory = connectorFactory }; // Open log file if specified diff --git a/src/DemaConsulting.BuildMark/DemaConsulting.BuildMark.csproj b/src/DemaConsulting.BuildMark/DemaConsulting.BuildMark.csproj index 1507c31..48844c6 100644 --- a/src/DemaConsulting.BuildMark/DemaConsulting.BuildMark.csproj +++ b/src/DemaConsulting.BuildMark/DemaConsulting.BuildMark.csproj @@ -47,6 +47,7 @@ + diff --git a/src/DemaConsulting.BuildMark/Program.cs b/src/DemaConsulting.BuildMark/Program.cs index 55f35da..5926d1e 100644 --- a/src/DemaConsulting.BuildMark/Program.cs +++ b/src/DemaConsulting.BuildMark/Program.cs @@ -155,8 +155,8 @@ private static void PrintHelp(Context context) /// The context containing command line arguments and program state. private static void ProcessBuildNotes(Context context) { - // Create repository connector - var connector = RepoConnectorFactory.Create(); + // Create repository connector using factory if provided, otherwise use default + var connector = context.ConnectorFactory?.Invoke() ?? RepoConnectorFactory.Create(); // Parse build version if provided DemaConsulting.BuildMark.Version? buildVersion = null; diff --git a/src/DemaConsulting.BuildMark/RepoConnectors/GitHubRepoConnector.cs b/src/DemaConsulting.BuildMark/RepoConnectors/GitHubRepoConnector.cs index 69d4d76..ec0051c 100644 --- a/src/DemaConsulting.BuildMark/RepoConnectors/GitHubRepoConnector.cs +++ b/src/DemaConsulting.BuildMark/RepoConnectors/GitHubRepoConnector.cs @@ -99,8 +99,8 @@ public override async Task GetBuildInformationAsync(Version? v // Build version tags from version and hash info var currentTag = new VersionTag(toVersion, toHash); - var baselineTag = fromVersion != null && fromHash != null - ? new VersionTag(fromVersion, fromHash) + var baselineTag = fromVersion != null && fromHash != null + ? new VersionTag(fromVersion, fromHash) : null; // Generate full changelog link for GitHub diff --git a/src/DemaConsulting.BuildMark/RepoConnectors/MockRepoConnector.cs b/src/DemaConsulting.BuildMark/RepoConnectors/MockRepoConnector.cs index 3924928..da45e78 100644 --- a/src/DemaConsulting.BuildMark/RepoConnectors/MockRepoConnector.cs +++ b/src/DemaConsulting.BuildMark/RepoConnectors/MockRepoConnector.cs @@ -111,8 +111,8 @@ public override async Task GetBuildInformationAsync(Version? v // Build version tags from version and hash info var currentTag = new VersionTag(toTagInfo, toHash.Trim()); - var baselineTag = fromTagInfo != null && fromHash != null - ? new VersionTag(fromTagInfo, fromHash.Trim()) + var baselineTag = fromTagInfo != null && fromHash != null + ? new VersionTag(fromTagInfo, fromHash.Trim()) : null; // Generate mock changelog link diff --git a/src/DemaConsulting.BuildMark/Validation.cs b/src/DemaConsulting.BuildMark/Validation.cs index 9139794..e48701d 100644 --- a/src/DemaConsulting.BuildMark/Validation.cs +++ b/src/DemaConsulting.BuildMark/Validation.cs @@ -18,20 +18,446 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +using System.Runtime.InteropServices; +using DemaConsulting.BuildMark.RepoConnectors; +using DemaConsulting.TestResults.IO; + namespace DemaConsulting.BuildMark; /// -/// Self-validation logic for BuildMark tool. +/// Provides self-validation functionality for the BuildMark tool. /// internal static class Validation { /// - /// Runs the self-validation logic. + /// Runs self-validation tests and optionally writes results to a file. /// - /// The context for output. + /// The context containing command line arguments and program state. public static void Run(Context context) { - // Stub implementation - to be fleshed out in a later PR - context.WriteLine("Self-validation is not yet implemented."); + // Print validation header + PrintValidationHeader(context); + + // Create test results collection + var testResults = new DemaConsulting.TestResults.TestResults + { + Name = "BuildMark Self-Validation" + }; + + // Create mock connector factory + var mockFactory = () => new MockRepoConnector() as IRepoConnector; + + // Run core functionality tests + RunMarkdownReportGeneration(context, testResults, mockFactory); + RunGitIntegration(context, testResults, mockFactory); + RunIssueTracking(context, testResults, mockFactory); + RunKnownIssuesReporting(context, testResults, mockFactory); + + // Calculate totals + var totalTests = testResults.Results.Count; + var passedTests = testResults.Results.Count(t => t.Outcome == DemaConsulting.TestResults.TestOutcome.Passed); + var failedTests = testResults.Results.Count(t => t.Outcome == DemaConsulting.TestResults.TestOutcome.Failed); + + // Print summary + context.WriteLine(""); + context.WriteLine($"Total Tests: {totalTests}"); + context.WriteLine($"Passed: {passedTests}"); + if (failedTests > 0) + { + context.WriteError($"Failed: {failedTests}"); + } + else + { + context.WriteLine($"Failed: {failedTests}"); + } + + // Write results file if requested + if (context.ResultsFile != null) + { + WriteResultsFile(context, testResults); + } + } + + /// + /// Prints the validation header with system information. + /// + /// The context for output. + private static void PrintValidationHeader(Context context) + { + context.WriteLine("# DEMA Consulting BuildMark Self-validation"); + context.WriteLine(""); + context.WriteLine("| Information | Value |"); + context.WriteLine("| :------------------ | :------------------------------------------------- |"); + context.WriteLine($"| BuildMark Version | {Program.Version,-50} |"); + context.WriteLine($"| Machine Name | {Environment.MachineName,-50} |"); + context.WriteLine($"| OS Version | {RuntimeInformation.OSDescription,-50} |"); + context.WriteLine($"| DotNet Runtime | {RuntimeInformation.FrameworkDescription,-50} |"); + context.WriteLine($"| Time Stamp | {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss} UTC{"",-29} |"); + context.WriteLine(""); + } + + /// + /// Runs a test for markdown report generation functionality. + /// + /// The context for output. + /// The test results collection. + /// The mock connector factory. + private static void RunMarkdownReportGeneration( + Context context, + DemaConsulting.TestResults.TestResults testResults, + Func mockFactory) + { + RunValidationTest( + context, + testResults, + "BuildMark_MarkdownReportGeneration", + "Markdown Report Generation Test", + mockFactory, + "build-report.md", + (logContent, reportContent) => + { + if (reportContent == null) + { + return "Report file not created"; + } + + if (reportContent.Contains("# Build Report") && + reportContent.Contains("## Version Information") && + reportContent.Contains("2.0.0") && + reportContent.Contains("current123hash456")) + { + return null; + } + + return "Report file missing expected content"; + }); + } + + /// + /// Runs a test for git integration functionality. + /// + /// The context for output. + /// The test results collection. + /// The mock connector factory. + private static void RunGitIntegration( + Context context, + DemaConsulting.TestResults.TestResults testResults, + Func mockFactory) + { + RunValidationTest( + context, + testResults, + "BuildMark_GitIntegration", + "Git Integration Test", + mockFactory, + null, + (logContent, _) => + { + if (logContent.Contains("Build Version: 2.0.0") && + logContent.Contains("Commit Hash: current123hash456") && + logContent.Contains("Previous Version: ver-1.1.0")) + { + return null; + } + + return "Expected git information not found in log"; + }); + } + + /// + /// Runs a test for issue tracking functionality. + /// + /// The context for output. + /// The test results collection. + /// The mock connector factory. + private static void RunIssueTracking( + Context context, + DemaConsulting.TestResults.TestResults testResults, + Func mockFactory) + { + RunValidationTest( + context, + testResults, + "BuildMark_IssueTracking", + "Issue Tracking Test", + mockFactory, + null, + (logContent, _) => + { + if (logContent.Contains("Changes: ") && + logContent.Contains("Bugs Fixed: ")) + { + return null; + } + + return "Expected issue tracking information not found in log"; + }); + } + + /// + /// Runs a test for known issues reporting functionality. + /// + /// The context for output. + /// The test results collection. + /// The mock connector factory. + private static void RunKnownIssuesReporting( + Context context, + DemaConsulting.TestResults.TestResults testResults, + Func mockFactory) + { + RunValidationTest( + context, + testResults, + "BuildMark_KnownIssuesReporting", + "Known Issues Reporting Test", + mockFactory, + "known-issues-report.md", + (logContent, reportContent) => + { + if (reportContent == null) + { + return "Report file not created"; + } + + if (logContent.Contains("Known Issues: 2") && + reportContent.Contains("## Known Issues")) + { + return null; + } + + return "Expected known issues not found in report"; + }); + } + + /// + /// Runs a validation test with common test execution logic. + /// + /// The context for output. + /// The test results collection. + /// The name of the test. + /// The display name for console output. + /// The mock connector factory. + /// Optional report file name to generate. + /// Function to validate test results. Returns null on success or error message on failure. + private static void RunValidationTest( + Context context, + DemaConsulting.TestResults.TestResults testResults, + string testName, + string displayName, + Func mockFactory, + string? reportFileName, + Func validator) + { + var startTime = DateTime.UtcNow; + var test = CreateTestResult(testName); + + try + { + using var tempDir = new TemporaryDirectory(); + var logFile = Path.Combine(tempDir.DirectoryPath, $"{testName}.log"); + var reportFile = reportFileName != null ? Path.Combine(tempDir.DirectoryPath, reportFileName) : null; + + // Build command line arguments + var args = new List + { + "--silent", + "--log", logFile, + "--build-version", "2.0.0" + }; + + if (reportFile != null) + { + args.Add("--report"); + args.Add(reportFile); + + // Include known issues if this is the known issues test + if (testName.Contains("KnownIssues")) + { + args.Add("--include-known-issues"); + } + } + + // Run the program + int exitCode; + using (var testContext = Context.Create([.. args], mockFactory)) + { + Program.Run(testContext); + exitCode = testContext.ExitCode; + } + + // Check if execution succeeded + if (exitCode == 0) + { + // Read log and report contents + var logContent = File.ReadAllText(logFile); + var reportContent = reportFile != null && File.Exists(reportFile) + ? File.ReadAllText(reportFile) + : null; + + // Validate the results + var errorMessage = validator(logContent, reportContent); + + if (errorMessage == null) + { + test.Outcome = DemaConsulting.TestResults.TestOutcome.Passed; + context.WriteLine($"✓ {displayName} - PASSED"); + } + else + { + test.Outcome = DemaConsulting.TestResults.TestOutcome.Failed; + test.ErrorMessage = errorMessage; + context.WriteError($"✗ {displayName} - FAILED: {errorMessage}"); + } + } + else + { + test.Outcome = DemaConsulting.TestResults.TestOutcome.Failed; + test.ErrorMessage = $"Program exited with code {exitCode}"; + context.WriteError($"✗ {displayName} - FAILED: Exit code {exitCode}"); + } + } + // Generic catch is justified here to handle any exception during test execution + catch (Exception ex) + { + HandleTestException(test, context, displayName, ex); + } + + FinalizeTestResult(test, startTime, testResults); + } + + /// + /// Writes test results to a file in TRX or JUnit format. + /// + /// The context for output. + /// The test results to write. + private static void WriteResultsFile(Context context, DemaConsulting.TestResults.TestResults testResults) + { + if (context.ResultsFile == null) + { + return; + } + + try + { + var extension = Path.GetExtension(context.ResultsFile).ToLowerInvariant(); + string content; + + if (extension == ".trx") + { + content = TrxSerializer.Serialize(testResults); + } + else if (extension == ".xml") + { + // Assume JUnit format for .xml extension + content = JUnitSerializer.Serialize(testResults); + } + else + { + context.WriteError($"Error: Unsupported results file format '{extension}'. Use .trx or .xml extension."); + return; + } + + File.WriteAllText(context.ResultsFile, content); + context.WriteLine($"Results written to {context.ResultsFile}"); + } + // Generic catch is justified here as a top-level handler to log file write errors + catch (Exception ex) + { + context.WriteError($"Error: Failed to write results file: {ex.Message}"); + } + } + + /// + /// Creates a new test result object with common properties. + /// + /// The name of the test. + /// A new test result object. + private static DemaConsulting.TestResults.TestResult CreateTestResult(string testName) + { + return new DemaConsulting.TestResults.TestResult + { + Name = testName, + ClassName = "Validation", + CodeBase = "BuildMark" + }; + } + + /// + /// Finalizes a test result by setting its duration and adding it to the collection. + /// + /// The test result to finalize. + /// The start time of the test. + /// The test results collection to add to. + private static void FinalizeTestResult( + DemaConsulting.TestResults.TestResult test, + DateTime startTime, + DemaConsulting.TestResults.TestResults testResults) + { + test.Duration = DateTime.UtcNow - startTime; + testResults.Results.Add(test); + } + + /// + /// Handles test exceptions by setting failure information and logging the error. + /// + /// The test result to update. + /// The context for output. + /// The display name for console output. + /// The exception that occurred. + private static void HandleTestException( + DemaConsulting.TestResults.TestResult test, + Context context, + string displayName, + Exception ex) + { + test.Outcome = DemaConsulting.TestResults.TestOutcome.Failed; + test.ErrorMessage = $"Exception: {ex.Message}"; + context.WriteError($"✗ {displayName} - FAILED: {ex.Message}"); + } + + /// + /// Represents a temporary directory that is automatically deleted when disposed. + /// + private sealed class TemporaryDirectory : IDisposable + { + /// + /// Gets the path to the temporary directory. + /// + public string DirectoryPath { get; } + + /// + /// Initializes a new instance of the class. + /// + public TemporaryDirectory() + { + DirectoryPath = Path.Combine(Path.GetTempPath(), $"buildmark_validation_{Guid.NewGuid()}"); + + try + { + Directory.CreateDirectory(DirectoryPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) + { + throw new InvalidOperationException($"Failed to create temporary directory: {ex.Message}", ex); + } + } + + /// + /// Deletes the temporary directory and all its contents. + /// + public void Dispose() + { + try + { + if (Directory.Exists(DirectoryPath)) + { + Directory.Delete(DirectoryPath, true); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Ignore cleanup errors during disposal + } + } } } diff --git a/test/DemaConsulting.BuildMark.Tests/BuildInformationTests.cs b/test/DemaConsulting.BuildMark.Tests/BuildInformationTests.cs index 3f4669f..65f4329 100644 --- a/test/DemaConsulting.BuildMark.Tests/BuildInformationTests.cs +++ b/test/DemaConsulting.BuildMark.Tests/BuildInformationTests.cs @@ -423,18 +423,18 @@ public async Task BuildInformation_ToMarkdown_UsesCorrectTableWidths() // Assert - verify table separators use correct width format (10:1 ratio with centered Issue) // The separator should have centered Issue column (:-:) and wide left-aligned Title column (:----------) - + // Verify the table separator appears in Changes section var changesStart = markdown.IndexOf("## Changes", StringComparison.Ordinal); var bugsStart = markdown.IndexOf("## Bugs Fixed", StringComparison.Ordinal); var changesSection = markdown.Substring(changesStart, bugsStart - changesStart); Assert.Contains("| :-: | :---------- |", changesSection); - + // Verify the table separator appears in Bugs Fixed section var knownIssuesStart = markdown.IndexOf("## Known Issues", StringComparison.Ordinal); var bugsSection = markdown.Substring(bugsStart, knownIssuesStart - bugsStart); Assert.Contains("| :-: | :---------- |", bugsSection); - + // Verify the table separator appears in Known Issues section var knownIssuesSection = markdown.Substring(knownIssuesStart); Assert.Contains("| :-: | :---------- |", knownIssuesSection); diff --git a/test/DemaConsulting.BuildMark.Tests/IntegrationTests.cs b/test/DemaConsulting.BuildMark.Tests/IntegrationTests.cs new file mode 100644 index 0000000..318eed1 --- /dev/null +++ b/test/DemaConsulting.BuildMark.Tests/IntegrationTests.cs @@ -0,0 +1,255 @@ +// Copyright (c) DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +namespace DemaConsulting.BuildMark.Tests; + +/// +/// Integration tests that run the BuildMark application through dotnet. +/// +[TestClass] +public class IntegrationTests +{ + private string _dllPath = string.Empty; + + /// + /// Initialize test by locating the BuildMark DLL. + /// + [TestInitialize] + public void TestInitialize() + { + // The DLL should be in the same directory as the test assembly + // because the test project references the main project + var baseDir = AppContext.BaseDirectory; + _dllPath = Path.Combine(baseDir, "DemaConsulting.BuildMark.dll"); + + Assert.IsTrue(File.Exists(_dllPath), $"Could not find BuildMark DLL at {_dllPath}"); + } + + /// + /// Test that version flag outputs version information. + /// + [TestMethod] + public void IntegrationTest_VersionFlag_OutputsVersion() + { + // Run the application with --version flag + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--version"); + + // Verify success + Assert.AreEqual(0, exitCode); + + // Verify version is output + Assert.IsFalse(string.IsNullOrWhiteSpace(output)); + Assert.DoesNotContain("Error", output); + } + + /// + /// Test that help flag outputs usage information. + /// + [TestMethod] + public void IntegrationTest_HelpFlag_OutputsUsageInformation() + { + // Run the application with --help flag + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--help"); + + // Verify success + Assert.AreEqual(0, exitCode); + + // Verify usage information + Assert.Contains("Usage: buildmark", output); + Assert.Contains("Options:", output); + Assert.Contains("--version", output); + Assert.Contains("--help", output); + } + + /// + /// Test that silent flag suppresses output. + /// + [TestMethod] + public void IntegrationTest_SilentFlag_SuppressesOutput() + { + // Run the application with --silent and --help flags + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--silent", + "--help"); + + // Verify success + Assert.AreEqual(0, exitCode); + + // Verify no banner in output + Assert.DoesNotContain("BuildMark version", output); + } + + /// + /// Test that invalid argument shows error. + /// + [TestMethod] + public void IntegrationTest_InvalidArgument_ShowsError() + { + // Run the application with invalid argument + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--invalid-argument"); + + // Verify error exit code + Assert.AreEqual(1, exitCode); + + // Verify error message + Assert.Contains("Error:", output); + Assert.Contains("Unsupported argument", output); + } + + /// + /// Test that validate flag runs self-validation. + /// + [TestMethod] + public void IntegrationTest_ValidateFlag_RunsSelfValidation() + { + // Run the application with --validate flag + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--validate"); + + // Verify success + Assert.AreEqual(0, exitCode); + + // Verify validation runs + Assert.IsFalse(string.IsNullOrWhiteSpace(output)); + } + + /// + /// Test that log parameter is accepted. + /// + [TestMethod] + public void IntegrationTest_LogParameter_IsAccepted() + { + // Run the application with log parameter + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--log", "test.log", + "--help"); + + // Verify success + Assert.AreEqual(0, exitCode); + + // Verify it's not an argument error + Assert.DoesNotContain("Unsupported argument", output); + } + + /// + /// Test that report parameter is accepted. + /// + [TestMethod] + public void IntegrationTest_ReportParameter_IsAccepted() + { + // Run the application with report parameter + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--report", "output.md", + "--help"); + + // Verify success + Assert.AreEqual(0, exitCode); + + // Verify it's not an argument error + Assert.DoesNotContain("Unsupported argument", output); + } + + /// + /// Test that report-depth parameter is accepted. + /// + [TestMethod] + public void IntegrationTest_ReportDepthParameter_IsAccepted() + { + // Run the application with report-depth parameter + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--report-depth", "2", + "--help"); + + // Verify success + Assert.AreEqual(0, exitCode); + + // Verify it's not an argument error + Assert.DoesNotContain("Unsupported argument", output); + } + + /// + /// Test that build-version parameter is accepted. + /// + [TestMethod] + public void IntegrationTest_BuildVersionParameter_IsAccepted() + { + // Run the application with build-version parameter + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--build-version", "1.0.0", + "--help"); + + // Verify success + Assert.AreEqual(0, exitCode); + + // Verify it's not an argument error + Assert.DoesNotContain("Unsupported argument", output); + } + + /// + /// Test that results parameter is accepted. + /// + [TestMethod] + public void IntegrationTest_ResultsParameter_IsAccepted() + { + // Run the application with results parameter + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--results", "results.trx", + "--help"); + + // Verify success + Assert.AreEqual(0, exitCode); + + // Verify it's not an argument error + Assert.DoesNotContain("Unsupported argument", output); + } +} diff --git a/test/DemaConsulting.BuildMark.Tests/Runner.cs b/test/DemaConsulting.BuildMark.Tests/Runner.cs new file mode 100644 index 0000000..914eb74 --- /dev/null +++ b/test/DemaConsulting.BuildMark.Tests/Runner.cs @@ -0,0 +1,72 @@ +// Copyright (c) DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +using System.Diagnostics; + +namespace DemaConsulting.BuildMark.Tests; + +/// +/// Program runner class for integration testing. +/// +internal static class Runner +{ + /// + /// Runs the specified program and captures its output. + /// + /// Program output (stdout and stderr combined). + /// Program name or path. + /// Program arguments. + /// Program exit code. + /// Thrown when process fails to start. + public static int Run(out string output, string program, params string[] arguments) + { + // Construct the start information + var startInfo = new ProcessStartInfo(program) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + // Add the arguments + foreach (var argument in arguments) + { + startInfo.ArgumentList.Add(argument); + } + + // Start the process + using var process = Process.Start(startInfo) ?? + throw new InvalidOperationException("Failed to start process"); + + // Read output asynchronously to avoid buffer overflow + var outputTask = process.StandardOutput.ReadToEndAsync(); + var errorTask = process.StandardError.ReadToEndAsync(); + + // Wait for the process to exit + process.WaitForExit(); + + // Combine stdout and stderr, save the output and return the exit code + var stdout = outputTask.GetAwaiter().GetResult(); + var stderr = errorTask.GetAwaiter().GetResult(); + output = stdout + stderr; + return process.ExitCode; + } +}