Introduce mutation testing (App-vNext#1031)

Author: martintmk

.github/workflows/build.yml

@@ -52,6 +52,12 @@ jobs:
         files: ./src/Polly.Specs/coverage-reports/Cobertura.xml,./src/Polly.Core.Tests/coverage-reports/Cobertura.xml
         flags: ${{ matrix.os_name }}
 
+    - name: Upload Mutation Report
+      uses: actions/upload-artifact@v3
+      with:
+        name: mutation-report-${{ matrix.os_name }}
+        path: StrykerOutput
+
     - name: Publish NuGet packages
       uses: actions/upload-artifact@v3
       with:

build.cake

@@ -10,6 +10,7 @@ var configuration = Argument<string>("configuration", "Release");
 //////////////////////////////////////////////////////////////////////
 
 #Tool "xunit.runner.console&version=2.4.2"
+#Tool "dotnet-stryker&version=3.6.1"
 
 //////////////////////////////////////////////////////////////////////
 // EXTERNAL NUGET LIBRARIES
@@ -46,6 +47,10 @@ string nugetVersion;
 string assemblyVersion;
 string assemblySemver;
 
+// Stryker / Mutation Testing
+var strykerConfig = File("./eng/stryker-config.json");
+var strykerOutput = Directory("StrykerOutput");
+
 ///////////////////////////////////////////////////////////////////////////////
 // INNER CLASSES
 ///////////////////////////////////////////////////////////////////////////////
@@ -84,7 +89,8 @@ Task("__Clean")
     {
         testResultsDir,
         nupkgDestDir,
-        artifactsDir
+        artifactsDir,
+        strykerOutput
   	};
 
     CleanDirectories(cleanDirectories);
@@ -206,6 +212,27 @@ Task("__RunTests")
     }
 });
 
+Task("__RunMutationTests")
+    .Does(() =>
+{
+    TestProject(File("./src/Polly/Polly.csproj"), File("./src/Polly.Specs/Polly.Specs.csproj"), "Polly");
+    TestProject(File("./src/Polly.Core/Polly.Core.csproj"), File("./src/Polly.Core.Tests/Polly.Core.Tests.csproj"), "Polly.Core");
+
+    void TestProject(FilePath proj, FilePath testProj, string project)
+    {
+        var strykerPath = Context.Tools.Resolve("Stryker.CLI.dll");
+        var mutationScore = XmlPeek(proj, "/Project/PropertyGroup/MutationScore/text()", new XmlPeekSettings { SuppressWarning = true });
+        var score = int.Parse(mutationScore);
+
+        Information($"Running mutation tests for '{proj}'. Test Project: '{testProj}'");
+        var result = StartProcess("dotnet", $"{strykerPath} --project {project} --test-project {testProj} --break-at {score} --config-file {strykerConfig} --output {strykerOutput}/{project}");
+        if (result != 0)
+        {
+            throw new InvalidOperationException($"The mutation testing of '{project}' project failed.");
+        }
+    }
+});
+
 Task("__CreateSignedNuGetPackages")
     .Does(() =>
 {
@@ -240,6 +267,7 @@ Task("Build")
     .IsDependentOn("__UpdateAssemblyVersionInformation")
     .IsDependentOn("__BuildSolutions")
     .IsDependentOn("__RunTests")
+    .IsDependentOn("__RunMutationTests")
     .IsDependentOn("__CreateSignedNuGetPackages");
 
 ///////////////////////////////////////////////////////////////////////////////

eng/Common.targets

@@ -4,7 +4,6 @@
     <SignAssembly>true</SignAssembly>
     <LangVersion>latest</LangVersion>
     <ImplicitUsings>enable</ImplicitUsings>
-    <RootDir>$(MSBuildThisFileDirectory)..\</RootDir>
     <ContinuousIntegrationBuild Condition=" '$(CI)' != '' ">true</ContinuousIntegrationBuild>
   </PropertyGroup>
 

eng/stryker-config.json

@@ -0,0 +1,26 @@
+{
+  "stryker-config": {
+    "reporters": [
+      "json",
+      "html",
+      "markdown"
+    ],
+    "ignore-methods": [
+      "*Exception.ctor",
+      "AddError",
+      "ConfigureAwait",
+      "Dispose",
+      "LogError",
+      "LogInformation"
+    ],
+    "ignore-mutations": [
+      "block",
+      "statement"
+    ],
+    "target-framework": "net7.0",
+    "thresholds": {
+      "high": 100,
+      "low": 100
+    }
+  }
+}

src/Directory.Build.props

@@ -1,6 +1,4 @@
 <Project>
-  <Import Project="../eng/Common.targets" />
-
   <ItemGroup>
     <Using Include="System.Collections" />
     <Using Include="System.Collections.Concurrent" />

src/Directory.Build.targets

@@ -1,4 +1,5 @@
 <Project>
-  <Import Project="$(RootDir)eng/Test.targets" Condition="$(ProjectType) == 'Test'" />
-  <Import Project="$(RootDir)eng/Library.targets" Condition="$(ProjectType) == 'Library'" />
+  <Import Project="$(MsBuildThisFileDirectory)../eng/Common.targets" />
+  <Import Project="$(MsBuildThisFileDirectory)../eng/Test.targets" Condition="$(ProjectType) == 'Test'" />
+  <Import Project="$(MsBuildThisFileDirectory)../eng/Library.targets" Condition="$(ProjectType) == 'Library'" />
 </Project>

src/Polly.Core.Tests/DummyTest.cs

@@ -0,0 +1,12 @@
+using Xunit;
+
+namespace Polly.Core.Tests;
+
+public class DummyTest
+{
+    [Fact]
+    public void Success()
+    {
+        Assert.True(true);
+    }
+}

src/Polly.Core/Polly.Core.csproj

@@ -9,6 +9,7 @@
     <ProjectType>Library</ProjectType>
     <UseDefaultAnalyzers>true</UseDefaultAnalyzers>
     <SkipPollyUsings>true</SkipPollyUsings>
+    <MutationScore>100</MutationScore>
   </PropertyGroup>
 
   <ItemGroup>

src/Polly/Polly.csproj

@@ -4,6 +4,7 @@
     <TargetFrameworks>netstandard1.1;netstandard2.0;net461;net472</TargetFrameworks>
     <AssemblyTitle>Polly</AssemblyTitle>
     <ProjectType>Library</ProjectType>
+    <MutationScore>70</MutationScore>
   </PropertyGroup>
 
   <PropertyGroup Condition=" '$(TargetFramework)' == 'netstandard1.1' ">