From 3b4406161b259fe5f4b433ac08cfdb8ca9afd059 Mon Sep 17 00:00:00 2001 From: Ava S Date: Tue, 30 Aug 2022 12:17:35 +0200 Subject: [PATCH 01/41] adding support for a service container docker logs --- .../Container/DockerCommandManager.cs | 6 +++ .../ContainerOperationProvider.cs | 16 +++++-- .../L0/Worker/ContainerOperationProviderL0.cs | 44 +++++++++++++++++++ 3 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 src/Test/L0/Worker/ContainerOperationProviderL0.cs diff --git a/src/Runner.Worker/Container/DockerCommandManager.cs b/src/Runner.Worker/Container/DockerCommandManager.cs index a0c158bdf68..265cafc7576 100644 --- a/src/Runner.Worker/Container/DockerCommandManager.cs +++ b/src/Runner.Worker/Container/DockerCommandManager.cs @@ -32,6 +32,7 @@ public interface IDockerCommandManager : IRunnerService Task DockerExec(IExecutionContext context, string containerId, string options, string command); Task DockerExec(IExecutionContext context, string containerId, string options, string command, List outputs); Task> DockerInspect(IExecutionContext context, string dockerObject, string options); + Task> DockerInspectLogs(IExecutionContext context, string dockerContainerId); Task> DockerPort(IExecutionContext context, string containerId); Task DockerLogin(IExecutionContext context, string configFileDirectory, string registry, string username, string password); } @@ -352,6 +353,11 @@ public async Task> DockerInspect(IExecutionContext context, string return await ExecuteDockerCommandAsync(context, "inspect", $"{options} {dockerObject}"); } + public async Task> DockerInspectLogs(IExecutionContext context, string dockerContainerId) + { + return await ExecuteDockerCommandAsync(context, "logs", $"{dockerContainerId}"); + } + public async Task> DockerPort(IExecutionContext context, string containerId) { List portMappingLines = await ExecuteDockerCommandAsync(context, "port", containerId); diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 73472795c5e..0b15d54c643 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -101,7 +101,8 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec executionContext.Output("##[group]Waiting for all services to be ready"); foreach (var container in containers.Where(c => !c.IsJobContainer)) { - await ContainerHealthcheck(executionContext, container); + var healthcheck = await Healthcheck(executionContext, container); + await ContainerHealthcheckLogs(executionContext, container, healthcheck); } executionContext.Output("##[endgroup]"); } @@ -395,14 +396,13 @@ private async Task RemoveContainerNetworkAsync(IExecutionContext executionContex } } - private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container) - { + private async Task Healthcheck(IExecutionContext executionContext, ContainerInfo container){ string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\""; string serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); if (string.IsNullOrEmpty(serviceHealth)) { // Container has no HEALTHCHECK - return; + return String.Empty; } var retryCount = 0; while (string.Equals(serviceHealth, "starting", StringComparison.OrdinalIgnoreCase)) @@ -413,12 +413,20 @@ private async Task ContainerHealthcheck(IExecutionContext executionContext, Cont serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); retryCount++; } + return serviceHealth; + } + + private async Task ContainerHealthcheckLogs(IExecutionContext executionContext, ContainerInfo container, string serviceHealth) + { + if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase)) { executionContext.Output($"{container.ContainerNetworkAlias} service is healthy."); } else { + List dockerLogs = await _dockerManager.DockerInspectLogs(context: executionContext, dockerContainerId: container.ContainerId); + dockerLogs.ForEach(log => executionContext.Output(log)); throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}."); } } diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs new file mode 100644 index 00000000000..f4df2c83a14 --- /dev/null +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -0,0 +1,44 @@ +using System; +using GitHub.Runner.Worker; +using GitHub.Runner.Worker.Container; +using Moq; +using Xunit; + +namespace GitHub.Runner.Common.Tests.Worker +{ + public sealed class ContainerOperationProviderL0 + { + // private Mock _commandManager; + // private ContainerOperationProvider operationProvider; + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void EnablePluginInternalCommand() + { + + // _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())) + // .Returns((string tag, string line) => + // { + // hc.GetTrace().Info($"{tag} {line}"); + // return 1; + // }); + // _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + // .Callback((Issue issue, string message) => + // { + // hc.GetTrace().Info($"{issue.Type} {issue.Message} {message ?? string.Empty}"); + // }); + + // _commandManager.EnablePluginInternalCommand(); + + // Assert.True(_commandManager.TryProcessCommand(_ec.Object, "##[internal-set-repo-path repoFullName=actions/runner;workspaceRepo=true]somepath", null)); + + // _pipelineDirectoryManager.Verify(x => x.UpdateRepositoryDirectory(_ec.Object, "actions/runner", "somepath", true), Times.Once); + + } + // private TestHostContext CreateTestContext([CallerMemberName] string testName = "") { + // return null; + // } + + } +} \ No newline at end of file From 2e8d8a74aba1dad54a863dffc4e7140f651aaeb1 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Wed, 31 Aug 2022 10:05:44 +0000 Subject: [PATCH 02/41] Adding Unit test to ContainerOperationProvider --- .../ContainerOperationProvider.cs | 4 +- src/Test/L0/TestHostContext.cs | 3 +- .../L0/Worker/ContainerOperationProviderL0.cs | 116 ++++++++++++++---- 3 files changed, 93 insertions(+), 30 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 0b15d54c643..0a76f8aca83 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -396,7 +396,7 @@ private async Task RemoveContainerNetworkAsync(IExecutionContext executionContex } } - private async Task Healthcheck(IExecutionContext executionContext, ContainerInfo container){ + public async Task Healthcheck(IExecutionContext executionContext, ContainerInfo container){ string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\""; string serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); if (string.IsNullOrEmpty(serviceHealth)) @@ -416,7 +416,7 @@ private async Task Healthcheck(IExecutionContext executionContext, Conta return serviceHealth; } - private async Task ContainerHealthcheckLogs(IExecutionContext executionContext, ContainerInfo container, string serviceHealth) + public async Task ContainerHealthcheckLogs(IExecutionContext executionContext, ContainerInfo container, string serviceHealth) { if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase)) diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index ce1ec01bcbe..b31f228ca85 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -12,7 +12,7 @@ using GitHub.DistributedTask.Logging; using System.Net.Http.Headers; using GitHub.Runner.Sdk; - +using GitHub.Runner.Worker.Container; namespace GitHub.Runner.Common.Tests { public sealed class TestHostContext : IHostContext, IDisposable @@ -26,6 +26,7 @@ public sealed class TestHostContext : IHostContext, IDisposable private string _suiteName; private string _testName; private Tracing _trace; + private AssemblyLoadContext _loadContext; private string _tempDirectoryRoot = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("D")); private StartupType _startupType; diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index f4df2c83a14..61379aebb48 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -1,44 +1,106 @@ -using System; using GitHub.Runner.Worker; using GitHub.Runner.Worker.Container; -using Moq; using Xunit; +using Moq; +using GitHub.Runner.Worker.Container.ContainerHooks; +using System.Threading.Tasks; +using System.Collections.Generic; +using System; +using System.IO; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; + namespace GitHub.Runner.Common.Tests.Worker { + public sealed class ContainerOperationProviderL0 { - // private Mock _commandManager; - // private ContainerOperationProvider operationProvider; + + private TestHostContext _hc; + private Mock _ec; + private Mock _dockerManager; + private Mock _containerHookManager; + + private ContainerOperationProvider containerOperationProvider; + + private List healthyDockerStatus = new List { "healthy" }; + private List dockerLogs = new List { "log1", "log2", "log3" }; [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void EnablePluginInternalCommand() + public async void Healthchecktest_healthyDocker() { - - // _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())) - // .Returns((string tag, string line) => - // { - // hc.GetTrace().Info($"{tag} {line}"); - // return 1; - // }); - // _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) - // .Callback((Issue issue, string message) => - // { - // hc.GetTrace().Info($"{issue.Type} {issue.Message} {message ?? string.Empty}"); - // }); - - // _commandManager.EnablePluginInternalCommand(); - - // Assert.True(_commandManager.TryProcessCommand(_ec.Object, "##[internal-set-repo-path repoFullName=actions/runner;workspaceRepo=true]somepath", null)); - - // _pipelineDirectoryManager.Verify(x => x.UpdateRepositoryDirectory(_ec.Object, "actions/runner", "somepath", true), Times.Once); - + Setup(); + + var containerInfo = new ContainerInfo() { ContainerImage = "ubuntu:16.04" }; + + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(healthyDockerStatus)); + var result = await containerOperationProvider.Healthcheck(_ec.Object, containerInfo); + + _dockerManager.Verify(dm => dm.DockerInspectLogs(It.IsAny(), It.IsAny()), Times.Never()); + } - // private TestHostContext CreateTestContext([CallerMemberName] string testName = "") { - // return null; - // } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void Healthchecktest_dockerError() + { + Setup(); + + var containerInfo = new ContainerInfo() { ContainerImage = "ubuntu:16.04", ContainerId = "1234" }; + + _dockerManager.Setup(x => x.DockerInspectLogs(_ec.Object, containerInfo.ContainerId)).Returns(Task.FromResult(dockerLogs)); + + await Assert.ThrowsAsync(() => containerOperationProvider.ContainerHealthcheckLogs(_ec.Object, containerInfo, "error")); + + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void Healthchecktest_dockerError_inspectLogs() + { + Setup(); + + var containerInfo = new ContainerInfo() { ContainerImage = "ubuntu:16.04", ContainerId = "1234" }; + + _dockerManager.Setup(x => x.DockerInspectLogs(_ec.Object, containerInfo.ContainerId)).Returns(Task.FromResult(dockerLogs)); + + try + { + await containerOperationProvider.ContainerHealthcheckLogs(_ec.Object, containerInfo, "error"); + + } + catch (InvalidOperationException) + { + //TODO validate the log message written to the _ec + } + + } + + private void Setup([CallerMemberName] string testName = "") + { + _hc = new TestHostContext(this, "name"); + _ec = new Mock(); + + _dockerManager = new Mock(); + _containerHookManager = new Mock(); + containerOperationProvider = new ContainerOperationProvider(); + + _hc.SetSingleton(_dockerManager.Object); + _hc.SetSingleton(_containerHookManager.Object); + + var list = new List(); + list.Add("result"); + + _ec.Setup(x => x.Global).Returns(new GlobalContext()); + + containerOperationProvider.Initialize(_hc); + } } } \ No newline at end of file From 160d07e576d2bbd5e703fbb4a54c31da77486fc9 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Wed, 31 Aug 2022 13:55:18 +0000 Subject: [PATCH 03/41] Adding another test to ContainerOperationProvider --- .../L0/Worker/ContainerOperationProviderL0.cs | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index 61379aebb48..98a5db1aee7 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -23,24 +23,30 @@ public sealed class ContainerOperationProviderL0 private Mock _ec; private Mock _dockerManager; private Mock _containerHookManager; - private ContainerOperationProvider containerOperationProvider; + private Mock serverQueue; + + private Mock pagingLogger; + private List healthyDockerStatus = new List { "healthy" }; private List dockerLogs = new List { "log1", "log2", "log3" }; + private ContainerInfo containerInfo; + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] public async void Healthchecktest_healthyDocker() { + //Arrange Setup(); - - var containerInfo = new ContainerInfo() { ContainerImage = "ubuntu:16.04" }; - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(healthyDockerStatus)); + + //Act var result = await containerOperationProvider.Healthcheck(_ec.Object, containerInfo); + //Assert _dockerManager.Verify(dm => dm.DockerInspectLogs(It.IsAny(), It.IsAny()), Times.Never()); } @@ -50,12 +56,12 @@ public async void Healthchecktest_healthyDocker() [Trait("Category", "Worker")] public async void Healthchecktest_dockerError() { + //Arrange Setup(); - - var containerInfo = new ContainerInfo() { ContainerImage = "ubuntu:16.04", ContainerId = "1234" }; - _dockerManager.Setup(x => x.DockerInspectLogs(_ec.Object, containerInfo.ContainerId)).Returns(Task.FromResult(dockerLogs)); + //Act + //Asert await Assert.ThrowsAsync(() => containerOperationProvider.ContainerHealthcheckLogs(_ec.Object, containerInfo, "error")); } @@ -65,33 +71,42 @@ public async void Healthchecktest_dockerError() [Trait("Category", "Worker")] public async void Healthchecktest_dockerError_inspectLogs() { + //Arrange Setup(); - - var containerInfo = new ContainerInfo() { ContainerImage = "ubuntu:16.04", ContainerId = "1234" }; - _dockerManager.Setup(x => x.DockerInspectLogs(_ec.Object, containerInfo.ContainerId)).Returns(Task.FromResult(dockerLogs)); try { + //Act await containerOperationProvider.ContainerHealthcheckLogs(_ec.Object, containerInfo, "error"); } catch (InvalidOperationException) { - //TODO validate the log message written to the _ec + + //Assert + _ec.Verify(pL => pL.Write(It.IsAny(), It.IsAny()), Times.Exactly(3)); + } } private void Setup([CallerMemberName] string testName = "") { + containerInfo = new ContainerInfo() { ContainerImage = "ubuntu:16.04" }; _hc = new TestHostContext(this, "name"); _ec = new Mock(); + serverQueue = new Mock(); + pagingLogger = new Mock(); _dockerManager = new Mock(); _containerHookManager = new Mock(); containerOperationProvider = new ContainerOperationProvider(); + _hc.SetSingleton(_dockerManager.Object); + _hc.SetSingleton(serverQueue.Object); + _hc.SetSingleton(pagingLogger.Object); + _hc.SetSingleton(_dockerManager.Object); _hc.SetSingleton(_containerHookManager.Object); From ae7bb3143161fcb469504a05e98274ff92b1993a Mon Sep 17 00:00:00 2001 From: Ava S Date: Tue, 6 Sep 2022 10:10:52 +0200 Subject: [PATCH 04/41] placed the docker logs output in dedicated ##group section --- src/Runner.Worker/ContainerOperationProvider.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 0a76f8aca83..5a0630e9f6f 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -425,8 +425,10 @@ public async Task ContainerHealthcheckLogs(IExecutionContext executionContext, C } else { + executionContext.Output($"##[group]Docker logs for container id {container.ContainerId}"); List dockerLogs = await _dockerManager.DockerInspectLogs(context: executionContext, dockerContainerId: container.ContainerId); dockerLogs.ForEach(log => executionContext.Output(log)); + executionContext.Output("##[endgroup]"); throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}."); } } From 892a90cc999d104ae6342a06222bd959b0736cab Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Tue, 6 Sep 2022 09:22:12 +0000 Subject: [PATCH 05/41] Removed the exception thrown if the service container was not healthy --- src/Runner.Worker/ContainerOperationProvider.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 5a0630e9f6f..21c82fcfde6 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -429,7 +429,6 @@ public async Task ContainerHealthcheckLogs(IExecutionContext executionContext, C List dockerLogs = await _dockerManager.DockerInspectLogs(context: executionContext, dockerContainerId: container.ContainerId); dockerLogs.ForEach(log => executionContext.Output(log)); executionContext.Output("##[endgroup]"); - throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}."); } } From d5332710159502b2c9a830bc081973bc6ad2d615 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Tue, 6 Sep 2022 09:51:30 +0000 Subject: [PATCH 06/41] Removed duplicated logging to the executionContext --- src/Runner.Worker/ContainerOperationProvider.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 21c82fcfde6..2a3256a580f 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -426,8 +426,7 @@ public async Task ContainerHealthcheckLogs(IExecutionContext executionContext, C else { executionContext.Output($"##[group]Docker logs for container id {container.ContainerId}"); - List dockerLogs = await _dockerManager.DockerInspectLogs(context: executionContext, dockerContainerId: container.ContainerId); - dockerLogs.ForEach(log => executionContext.Output(log)); + await _dockerManager.DockerInspectLogs(context: executionContext, dockerContainerId: container.ContainerId); executionContext.Output("##[endgroup]"); } } From 29e26c0aa1107be8f03dec92d0210e080cdea5e9 Mon Sep 17 00:00:00 2001 From: Ava S Date: Tue, 6 Sep 2022 17:21:01 +0200 Subject: [PATCH 07/41] Updated the container logs sub-section message --- src/Runner.Worker/ContainerOperationProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 2a3256a580f..a67fda95dea 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -425,7 +425,7 @@ public async Task ContainerHealthcheckLogs(IExecutionContext executionContext, C } else { - executionContext.Output($"##[group]Docker logs for container id {container.ContainerId}"); + executionContext.Output($"##[group] Container {container.ContainerImage} failed healthchecks, printing logs:"); await _dockerManager.DockerInspectLogs(context: executionContext, dockerContainerId: container.ContainerId); executionContext.Output("##[endgroup]"); } From 2f9ced96c6713f51390327e453689bbc368f7d58 Mon Sep 17 00:00:00 2001 From: AStancu Date: Tue, 6 Sep 2022 19:17:48 +0000 Subject: [PATCH 08/41] Print service containers only if they were healthy Unhealthy service logs are printed in ContainerHealthCheckLogs called prior to this step. --- .../ContainerOperationProvider.cs | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index a67fda95dea..ea065116e59 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -302,14 +302,18 @@ private async Task StopContainerAsync(IExecutionContext executionContext, Contai { if (!container.IsJobContainer) { - // Print logs for service container jobs (not the "action" job itself b/c that's already logged). - executionContext.Output($"Print service container logs: {container.ContainerDisplayName}"); - - int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId); - if (logsExitCode != 0) - { - executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); - } + var healthcheck = await Healthcheck(executionContext, container); + if (string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)){ + // Print logs for service container jobs (not the "action" job itself b/c that's already logged). + // Print them only if the service was healthy, else they were already logged via ContainerHealthCheckLogs. + executionContext.Output($"Print service container logs: {container.ContainerDisplayName}"); + + int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId); + if (logsExitCode != 0) + { + executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); + } + } } executionContext.Output($"Stop and remove container: {container.ContainerDisplayName}"); From 94e504c40e55a39956a2cae0449e2703d0f5722e Mon Sep 17 00:00:00 2001 From: Ava S Date: Tue, 6 Sep 2022 23:13:20 +0200 Subject: [PATCH 09/41] Removed recently added method to inspect docker logs The method was doing the same thing as the existing DockerLogs method. --- .../Container/DockerCommandManager.cs | 7 +--- .../ContainerOperationProvider.cs | 2 +- .../L0/Worker/ContainerOperationProviderL0.cs | 42 ++++--------------- 3 files changed, 10 insertions(+), 41 deletions(-) diff --git a/src/Runner.Worker/Container/DockerCommandManager.cs b/src/Runner.Worker/Container/DockerCommandManager.cs index 265cafc7576..c7b3ada12f0 100644 --- a/src/Runner.Worker/Container/DockerCommandManager.cs +++ b/src/Runner.Worker/Container/DockerCommandManager.cs @@ -32,7 +32,6 @@ public interface IDockerCommandManager : IRunnerService Task DockerExec(IExecutionContext context, string containerId, string options, string command); Task DockerExec(IExecutionContext context, string containerId, string options, string command, List outputs); Task> DockerInspect(IExecutionContext context, string dockerObject, string options); - Task> DockerInspectLogs(IExecutionContext context, string dockerContainerId); Task> DockerPort(IExecutionContext context, string containerId); Task DockerLogin(IExecutionContext context, string configFileDirectory, string registry, string username, string password); } @@ -352,11 +351,7 @@ public async Task> DockerInspect(IExecutionContext context, string { return await ExecuteDockerCommandAsync(context, "inspect", $"{options} {dockerObject}"); } - - public async Task> DockerInspectLogs(IExecutionContext context, string dockerContainerId) - { - return await ExecuteDockerCommandAsync(context, "logs", $"{dockerContainerId}"); - } + public async Task> DockerPort(IExecutionContext context, string containerId) { diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index ea065116e59..61b40b7a761 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -430,7 +430,7 @@ public async Task ContainerHealthcheckLogs(IExecutionContext executionContext, C else { executionContext.Output($"##[group] Container {container.ContainerImage} failed healthchecks, printing logs:"); - await _dockerManager.DockerInspectLogs(context: executionContext, dockerContainerId: container.ContainerId); + await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId); executionContext.Output("##[endgroup]"); } } diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index 98a5db1aee7..56a6f5e5492 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -47,48 +47,25 @@ public async void Healthchecktest_healthyDocker() var result = await containerOperationProvider.Healthcheck(_ec.Object, containerInfo); //Assert - _dockerManager.Verify(dm => dm.DockerInspectLogs(It.IsAny(), It.IsAny()), Times.Never()); + _dockerManager.Verify(dm => dm.DockerLogs(It.IsAny(), It.IsAny()), Times.Never()); } [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void Healthchecktest_dockerError() + public async void HealthcheckTestDockerErrorLogs() { //Arrange Setup(); - _dockerManager.Setup(x => x.DockerInspectLogs(_ec.Object, containerInfo.ContainerId)).Returns(Task.FromResult(dockerLogs)); - + _dockerManager.Setup(x => x.DockerLogs(_ec.Object, containerInfo.ContainerId)).Returns(Task.FromResult(1)); //Act - //Asert - await Assert.ThrowsAsync(() => containerOperationProvider.ContainerHealthcheckLogs(_ec.Object, containerInfo, "error")); - - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void Healthchecktest_dockerError_inspectLogs() - { - //Arrange - Setup(); - _dockerManager.Setup(x => x.DockerInspectLogs(_ec.Object, containerInfo.ContainerId)).Returns(Task.FromResult(dockerLogs)); - - try - { - //Act - await containerOperationProvider.ContainerHealthcheckLogs(_ec.Object, containerInfo, "error"); - - } - catch (InvalidOperationException) - { - - //Assert - _ec.Verify(pL => pL.Write(It.IsAny(), It.IsAny()), Times.Exactly(3)); - - } + await containerOperationProvider.ContainerHealthcheckLogs(_ec.Object, containerInfo, "error"); + //Assert + _ec.Verify(dm => dm.Write(It.IsAny(), It.IsAny()), Times.Exactly(2)); + _ec.Verify(dm => dm.Write(null, $"##[group] Container {containerInfo.ContainerImage} failed healthchecks, printing logs:"), Times.AtLeastOnce()); + _ec.Verify(dm => dm.Write(null, "##[endgroup]"), Times.AtLeastOnce()); } private void Setup([CallerMemberName] string testName = "") @@ -110,9 +87,6 @@ private void Setup([CallerMemberName] string testName = "") _hc.SetSingleton(_dockerManager.Object); _hc.SetSingleton(_containerHookManager.Object); - var list = new List(); - list.Add("result"); - _ec.Setup(x => x.Global).Returns(new GlobalContext()); containerOperationProvider.Initialize(_hc); From cfcf0831f3193369cf57dafe7061ea13c9608f02 Mon Sep 17 00:00:00 2001 From: AStancu Date: Tue, 6 Sep 2022 21:53:37 +0000 Subject: [PATCH 10/41] Added execution context error This will make a failed health check more visible in the UI without disrupting the execution of the program. --- src/Runner.Worker/ContainerOperationProvider.cs | 3 ++- src/Test/L0/Worker/ContainerOperationProviderL0.cs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 61b40b7a761..67f19f95df8 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -429,8 +429,9 @@ public async Task ContainerHealthcheckLogs(IExecutionContext executionContext, C } else { - executionContext.Output($"##[group] Container {container.ContainerImage} failed healthchecks, printing logs:"); + executionContext.Output($"##[group]Container {container.ContainerImage} failed healthchecks, printing logs:"); await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId); + executionContext.Error($"Failed to initialize container {container.ContainerImage}"); executionContext.Output("##[endgroup]"); } } diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index 56a6f5e5492..a1c3b70ccba 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -64,7 +64,7 @@ public async void HealthcheckTestDockerErrorLogs() //Assert _ec.Verify(dm => dm.Write(It.IsAny(), It.IsAny()), Times.Exactly(2)); - _ec.Verify(dm => dm.Write(null, $"##[group] Container {containerInfo.ContainerImage} failed healthchecks, printing logs:"), Times.AtLeastOnce()); + _ec.Verify(dm => dm.Write(null, $"##[group]Container {containerInfo.ContainerImage} failed healthchecks, printing logs:"), Times.AtLeastOnce()); _ec.Verify(dm => dm.Write(null, "##[endgroup]"), Times.AtLeastOnce()); } From 2d4dc37d49716453a3599a58c837c1f5c5b77b33 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Wed, 7 Sep 2022 08:39:57 +0000 Subject: [PATCH 11/41] Removing the section 'Waiting for all services to be ready' Since nested subsections are not being displayed properly and we already need one subsection per service error. --- src/Runner.Worker/ContainerOperationProvider.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 67f19f95df8..b712661b86e 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -98,13 +98,12 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec await StartContainerAsync(executionContext, container); } - executionContext.Output("##[group]Waiting for all services to be ready"); + executionContext.Output("Waiting for all services to be ready"); foreach (var container in containers.Where(c => !c.IsJobContainer)) { var healthcheck = await Healthcheck(executionContext, container); await ContainerHealthcheckLogs(executionContext, container, healthcheck); } - executionContext.Output("##[endgroup]"); } public async Task StopContainersAsync(IExecutionContext executionContext, object data) From ad8f17e9569cca24536b6b1557101d34f33bd1d6 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Wed, 7 Sep 2022 17:19:04 +0200 Subject: [PATCH 12/41] Update src/Runner.Worker/Container/DockerCommandManager.cs Co-authored-by: Tingluo Huang --- src/Runner.Worker/Container/DockerCommandManager.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Runner.Worker/Container/DockerCommandManager.cs b/src/Runner.Worker/Container/DockerCommandManager.cs index c7b3ada12f0..a0c158bdf68 100644 --- a/src/Runner.Worker/Container/DockerCommandManager.cs +++ b/src/Runner.Worker/Container/DockerCommandManager.cs @@ -351,7 +351,6 @@ public async Task> DockerInspect(IExecutionContext context, string { return await ExecuteDockerCommandAsync(context, "inspect", $"{options} {dockerObject}"); } - public async Task> DockerPort(IExecutionContext context, string containerId) { From 354c8bcbed0600fa2c2fcb471c9caf6089d62161 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Wed, 7 Sep 2022 17:19:21 +0200 Subject: [PATCH 13/41] Update src/Test/L0/TestHostContext.cs Co-authored-by: Tingluo Huang --- src/Test/L0/TestHostContext.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index b31f228ca85..92a0a93bc08 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -26,7 +26,6 @@ public sealed class TestHostContext : IHostContext, IDisposable private string _suiteName; private string _testName; private Tracing _trace; - private AssemblyLoadContext _loadContext; private string _tempDirectoryRoot = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("D")); private StartupType _startupType; From f0b315c9117e8bb9819111771822bf0adeaff395 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Thu, 8 Sep 2022 15:49:59 +0000 Subject: [PATCH 14/41] Change the logic for printing Service Containers logs Service container logs will be printed in the 'Start containers' section only if there is an error. Healthy services will have their logs printed in the 'Stop Containers' section. --- src/Runner.Worker/Container/ContainerInfo.cs | 2 ++ .../ContainerOperationProvider.cs | 34 +++++++++++++++---- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/Runner.Worker/Container/ContainerInfo.cs b/src/Runner.Worker/Container/ContainerInfo.cs index 9c114939e2d..292e1e332c1 100644 --- a/src/Runner.Worker/Container/ContainerInfo.cs +++ b/src/Runner.Worker/Container/ContainerInfo.cs @@ -92,6 +92,8 @@ public ContainerInfo(IHostContext hostContext, Pipelines.JobContainer container, public bool IsJobContainer { get; set; } public bool IsAlpine { get; set; } + public bool IsHealthy { get; set; } = true; + public IDictionary ContainerEnvironmentVariables { get diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index b712661b86e..ff84a9f2cc5 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -89,6 +89,7 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec executionContext.Output("##[group]Create local container network"); var containerNetwork = $"github_network_{Guid.NewGuid().ToString("N")}"; await CreateContainerNetworkAsync(executionContext, containerNetwork); + executionContext.JobContext.Container["network"] = new StringContextData(containerNetwork); executionContext.Output("##[endgroup]"); @@ -99,11 +100,27 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec } executionContext.Output("Waiting for all services to be ready"); + + bool IsAnyUnhealthy = false; foreach (var container in containers.Where(c => !c.IsJobContainer)) { + var healthcheck = await Healthcheck(executionContext, container); + if (!string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)) + { + IsAnyUnhealthy = true; + } await ContainerHealthcheckLogs(executionContext, container, healthcheck); } + if (IsAnyUnhealthy) throw new InvalidOperationException("One or more containers failed to start."); + } + + public void printHello() + { + Console.WriteLine("Hello World"); + // create an array of strings + + } public async Task StopContainersAsync(IExecutionContext executionContext, object data) @@ -299,10 +316,11 @@ private async Task StopContainerAsync(IExecutionContext executionContext, Contai if (!string.IsNullOrEmpty(container.ContainerId)) { - if (!container.IsJobContainer) + if (!container.IsJobContainer && container.IsHealthy) { var healthcheck = await Healthcheck(executionContext, container); - if (string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)){ + if (string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)) + { // Print logs for service container jobs (not the "action" job itself b/c that's already logged). // Print them only if the service was healthy, else they were already logged via ContainerHealthCheckLogs. executionContext.Output($"Print service container logs: {container.ContainerDisplayName}"); @@ -312,7 +330,7 @@ private async Task StopContainerAsync(IExecutionContext executionContext, Contai { executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); } - } + } } executionContext.Output($"Stop and remove container: {container.ContainerDisplayName}"); @@ -399,7 +417,8 @@ private async Task RemoveContainerNetworkAsync(IExecutionContext executionContex } } - public async Task Healthcheck(IExecutionContext executionContext, ContainerInfo container){ + public async Task Healthcheck(IExecutionContext executionContext, ContainerInfo container) + { string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\""; string serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); if (string.IsNullOrEmpty(serviceHealth)) @@ -421,17 +440,18 @@ public async Task Healthcheck(IExecutionContext executionContext, Contai public async Task ContainerHealthcheckLogs(IExecutionContext executionContext, ContainerInfo container, string serviceHealth) { - + if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase)) { executionContext.Output($"{container.ContainerNetworkAlias} service is healthy."); } else { - executionContext.Output($"##[group]Container {container.ContainerImage} failed healthchecks, printing logs:"); + + executionContext.Output($"Container {container.ContainerImage} failed healthchecks, printing logs:"); await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId); executionContext.Error($"Failed to initialize container {container.ContainerImage}"); - executionContext.Output("##[endgroup]"); + container.IsHealthy = false; } } From 2dc8f253598f22ed1667046423ee16400395e634 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Thu, 8 Sep 2022 15:52:13 +0000 Subject: [PATCH 15/41] Removed unused import --- src/Test/L0/TestHostContext.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index 92a0a93bc08..2ff297c998c 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -12,7 +12,6 @@ using GitHub.DistributedTask.Logging; using System.Net.Http.Headers; using GitHub.Runner.Sdk; -using GitHub.Runner.Worker.Container; namespace GitHub.Runner.Common.Tests { public sealed class TestHostContext : IHostContext, IDisposable From dfcbe9b1e18e34f41fcc29724bf80109f925cd3d Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Thu, 8 Sep 2022 15:59:06 +0000 Subject: [PATCH 16/41] Added back section group. --- src/Runner.Worker/ContainerOperationProvider.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index ff84a9f2cc5..2dff82e63e8 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -99,7 +99,7 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec await StartContainerAsync(executionContext, container); } - executionContext.Output("Waiting for all services to be ready"); + executionContext.Output("##[group]Waiting for all services to be ready"); bool IsAnyUnhealthy = false; foreach (var container in containers.Where(c => !c.IsJobContainer)) @@ -113,6 +113,7 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec await ContainerHealthcheckLogs(executionContext, container, healthcheck); } if (IsAnyUnhealthy) throw new InvalidOperationException("One or more containers failed to start."); + executionContext.Output("##[endgroup]"); } public void printHello() @@ -447,7 +448,6 @@ public async Task ContainerHealthcheckLogs(IExecutionContext executionContext, C } else { - executionContext.Output($"Container {container.ContainerImage} failed healthchecks, printing logs:"); await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId); executionContext.Error($"Failed to initialize container {container.ContainerImage}"); From 7a992f844df381a5281a890ae1d385bbb6f07446 Mon Sep 17 00:00:00 2001 From: AStancu Date: Fri, 9 Sep 2022 09:16:48 +0000 Subject: [PATCH 17/41] Moved service containers error logs to separate group sections --- .../Container/DockerCommandManager.cs | 3 ++ .../ContainerOperationProvider.cs | 44 ++++++++++--------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/Runner.Worker/Container/DockerCommandManager.cs b/src/Runner.Worker/Container/DockerCommandManager.cs index a0c158bdf68..8e32aeb8414 100644 --- a/src/Runner.Worker/Container/DockerCommandManager.cs +++ b/src/Runner.Worker/Container/DockerCommandManager.cs @@ -16,6 +16,7 @@ public interface IDockerCommandManager : IRunnerService { string DockerPath { get; } string DockerInstanceLabel { get; } + IList UnhealthyContainers {get; set; } Task DockerVersion(IExecutionContext context); Task DockerPull(IExecutionContext context, string image); Task DockerPull(IExecutionContext context, string image, string configFileDirectory); @@ -42,6 +43,8 @@ public class DockerCommandManager : RunnerService, IDockerCommandManager public string DockerInstanceLabel { get; private set; } + public IList UnhealthyContainers {get; set; } + public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 2dff82e63e8..ee2b51c1475 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -100,20 +100,33 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec } executionContext.Output("##[group]Waiting for all services to be ready"); - bool IsAnyUnhealthy = false; + _dockerManager.UnhealthyContainers = new List(); foreach (var container in containers.Where(c => !c.IsJobContainer)) { - var healthcheck = await Healthcheck(executionContext, container); if (!string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)) { - IsAnyUnhealthy = true; + IsAnyUnhealthy = true; + _dockerManager.UnhealthyContainers.Add(container); } - await ContainerHealthcheckLogs(executionContext, container, healthcheck); - } - if (IsAnyUnhealthy) throw new InvalidOperationException("One or more containers failed to start."); + else + { + executionContext.Output($"{container.ContainerNetworkAlias} service is healthy."); + } + } executionContext.Output("##[endgroup]"); + + if (IsAnyUnhealthy) + { + foreach (var container in _dockerManager.UnhealthyContainers) + { + executionContext.Output($"##[group]Service container {container.ContainerNetworkAlias} failed."); + await ContainerErrorLogs(executionContext, container); + executionContext.Output("##[endgroup]"); + } + throw new InvalidOperationException("One or more containers failed to start."); + } } public void printHello() @@ -323,7 +336,7 @@ private async Task StopContainerAsync(IExecutionContext executionContext, Contai if (string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)) { // Print logs for service container jobs (not the "action" job itself b/c that's already logged). - // Print them only if the service was healthy, else they were already logged via ContainerHealthCheckLogs. + // Print them only if the service was healthy, else they were already logged in the initialize containers section. executionContext.Output($"Print service container logs: {container.ContainerDisplayName}"); int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId); @@ -439,20 +452,11 @@ public async Task Healthcheck(IExecutionContext executionContext, Contai return serviceHealth; } - public async Task ContainerHealthcheckLogs(IExecutionContext executionContext, ContainerInfo container, string serviceHealth) + public async Task ContainerErrorLogs(IExecutionContext executionContext, ContainerInfo container) { - - if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase)) - { - executionContext.Output($"{container.ContainerNetworkAlias} service is healthy."); - } - else - { - executionContext.Output($"Container {container.ContainerImage} failed healthchecks, printing logs:"); - await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId); - executionContext.Error($"Failed to initialize container {container.ContainerImage}"); - container.IsHealthy = false; - } + await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId); + executionContext.Error($"Failed to initialize container {container.ContainerImage}"); + container.IsHealthy = false; } private async Task ContainerRegistryLogin(IExecutionContext executionContext, ContainerInfo container) From 76e4f51a21256d747c338f5bc3141174b26d44d5 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Fri, 9 Sep 2022 15:21:18 +0000 Subject: [PATCH 18/41] Removed the test testing the old logic flow. --- .../L0/Worker/ContainerOperationProviderL0.cs | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index a1c3b70ccba..7bd5675445c 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -5,13 +5,7 @@ using GitHub.Runner.Worker.Container.ContainerHooks; using System.Threading.Tasks; using System.Collections.Generic; -using System; -using System.IO; -using System.Reflection; using System.Runtime.CompilerServices; -using System.Text; -using System.Threading; - namespace GitHub.Runner.Common.Tests.Worker { @@ -51,22 +45,6 @@ public async void Healthchecktest_healthyDocker() } - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void HealthcheckTestDockerErrorLogs() - { - //Arrange - Setup(); - _dockerManager.Setup(x => x.DockerLogs(_ec.Object, containerInfo.ContainerId)).Returns(Task.FromResult(1)); - //Act - await containerOperationProvider.ContainerHealthcheckLogs(_ec.Object, containerInfo, "error"); - - //Assert - _ec.Verify(dm => dm.Write(It.IsAny(), It.IsAny()), Times.Exactly(2)); - _ec.Verify(dm => dm.Write(null, $"##[group]Container {containerInfo.ContainerImage} failed healthchecks, printing logs:"), Times.AtLeastOnce()); - _ec.Verify(dm => dm.Write(null, "##[endgroup]"), Times.AtLeastOnce()); - } private void Setup([CallerMemberName] string testName = "") { From 912d7d69323ee18b5f03b5188c529a6b0c03e6b9 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Tue, 13 Sep 2022 12:44:22 +0000 Subject: [PATCH 19/41] Remove unnecessary 'IsAnyUnhealthy' flag --- src/Runner.Worker/ContainerOperationProvider.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index ee2b51c1475..d809aa731d0 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -100,33 +100,33 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec } executionContext.Output("##[group]Waiting for all services to be ready"); - bool IsAnyUnhealthy = false; + _dockerManager.UnhealthyContainers = new List(); foreach (var container in containers.Where(c => !c.IsJobContainer)) { var healthcheck = await Healthcheck(executionContext, container); + if (!string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)) { - IsAnyUnhealthy = true; _dockerManager.UnhealthyContainers.Add(container); } else { executionContext.Output($"{container.ContainerNetworkAlias} service is healthy."); - } - } + } + } executionContext.Output("##[endgroup]"); - if (IsAnyUnhealthy) - { + if (_dockerManager.UnhealthyContainers.Count > 0) + { foreach (var container in _dockerManager.UnhealthyContainers) { executionContext.Output($"##[group]Service container {container.ContainerNetworkAlias} failed."); await ContainerErrorLogs(executionContext, container); - executionContext.Output("##[endgroup]"); + executionContext.Output("##[endgroup]"); } throw new InvalidOperationException("One or more containers failed to start."); - } + } } public void printHello() From f5461b78befd85e9a818d5b21b5faaab7a3ad468 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Tue, 13 Sep 2022 12:45:35 +0000 Subject: [PATCH 20/41] Remove printHello() function --- src/Runner.Worker/ContainerOperationProvider.cs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index d809aa731d0..103f1c36a59 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -129,14 +129,6 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec } } - public void printHello() - { - Console.WriteLine("Hello World"); - // create an array of strings - - - } - public async Task StopContainersAsync(IExecutionContext executionContext, object data) { Trace.Entering(); From 3c88e14ca5828237e5b6119d5bcfda42fc4823f0 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Tue, 13 Sep 2022 12:48:31 +0000 Subject: [PATCH 21/41] Add newline to TestHostContext --- src/Test/L0/TestHostContext.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index 2ff297c998c..ce1ec01bcbe 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -12,6 +12,7 @@ using GitHub.DistributedTask.Logging; using System.Net.Http.Headers; using GitHub.Runner.Sdk; + namespace GitHub.Runner.Common.Tests { public sealed class TestHostContext : IHostContext, IDisposable From 47ead99c13b3960507d68c28f7375e70812a2e8d Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Tue, 13 Sep 2022 13:47:58 +0000 Subject: [PATCH 22/41] Remove unnecessary field 'UnhealthyContainers' --- src/Runner.Worker/Container/DockerCommandManager.cs | 3 --- src/Runner.Worker/ContainerOperationProvider.cs | 8 ++++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Runner.Worker/Container/DockerCommandManager.cs b/src/Runner.Worker/Container/DockerCommandManager.cs index 8e32aeb8414..a0c158bdf68 100644 --- a/src/Runner.Worker/Container/DockerCommandManager.cs +++ b/src/Runner.Worker/Container/DockerCommandManager.cs @@ -16,7 +16,6 @@ public interface IDockerCommandManager : IRunnerService { string DockerPath { get; } string DockerInstanceLabel { get; } - IList UnhealthyContainers {get; set; } Task DockerVersion(IExecutionContext context); Task DockerPull(IExecutionContext context, string image); Task DockerPull(IExecutionContext context, string image, string configFileDirectory); @@ -43,8 +42,6 @@ public class DockerCommandManager : RunnerService, IDockerCommandManager public string DockerInstanceLabel { get; private set; } - public IList UnhealthyContainers {get; set; } - public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 103f1c36a59..3fbac07d955 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -101,14 +101,14 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec executionContext.Output("##[group]Waiting for all services to be ready"); - _dockerManager.UnhealthyContainers = new List(); + var unhealthyContainers = new List(); foreach (var container in containers.Where(c => !c.IsJobContainer)) { var healthcheck = await Healthcheck(executionContext, container); if (!string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)) { - _dockerManager.UnhealthyContainers.Add(container); + unhealthyContainers.Add(container); } else { @@ -117,9 +117,9 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec } executionContext.Output("##[endgroup]"); - if (_dockerManager.UnhealthyContainers.Count > 0) + if (unhealthyContainers.Count > 0) { - foreach (var container in _dockerManager.UnhealthyContainers) + foreach (var container in unhealthyContainers) { executionContext.Output($"##[group]Service container {container.ContainerNetworkAlias} failed."); await ContainerErrorLogs(executionContext, container); From b8aafc4ff1a737bfddfc7614583209509663f0cc Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Tue, 13 Sep 2022 13:51:52 +0000 Subject: [PATCH 23/41] Rename boolean flag indicating service container failure --- src/Runner.Worker/Container/ContainerInfo.cs | 2 +- src/Runner.Worker/ContainerOperationProvider.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Runner.Worker/Container/ContainerInfo.cs b/src/Runner.Worker/Container/ContainerInfo.cs index 292e1e332c1..ce883cc1096 100644 --- a/src/Runner.Worker/Container/ContainerInfo.cs +++ b/src/Runner.Worker/Container/ContainerInfo.cs @@ -92,7 +92,7 @@ public ContainerInfo(IHostContext hostContext, Pipelines.JobContainer container, public bool IsJobContainer { get; set; } public bool IsAlpine { get; set; } - public bool IsHealthy { get; set; } = true; + public bool FailedInitialization { get; set; } = false; public IDictionary ContainerEnvironmentVariables { diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 3fbac07d955..d77a50c707a 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -322,7 +322,7 @@ private async Task StopContainerAsync(IExecutionContext executionContext, Contai if (!string.IsNullOrEmpty(container.ContainerId)) { - if (!container.IsJobContainer && container.IsHealthy) + if (!container.IsJobContainer && !container.FailedInitialization) { var healthcheck = await Healthcheck(executionContext, container); if (string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)) @@ -448,7 +448,7 @@ public async Task ContainerErrorLogs(IExecutionContext executionContext, Contain { await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId); executionContext.Error($"Failed to initialize container {container.ContainerImage}"); - container.IsHealthy = false; + container.FailedInitialization = true; } private async Task ContainerRegistryLogin(IExecutionContext executionContext, ContainerInfo container) From 826cec2775e00b2ce8bf454bd4e86afc5b25fe16 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Wed, 14 Sep 2022 08:10:03 +0000 Subject: [PATCH 24/41] Refactor healthcheck logic to separate method to enable unit testing. --- .../ContainerOperationProvider.cs | 9 +++- .../L0/Worker/ContainerOperationProviderL0.cs | 52 +++++++++++++++---- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index d77a50c707a..b23ae388662 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -99,6 +99,11 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec await StartContainerAsync(executionContext, container); } + await RunContainersHealthcheck(executionContext, containers); + } + + public async Task RunContainersHealthcheck(IExecutionContext executionContext, List containers) + { executionContext.Output("##[group]Waiting for all services to be ready"); var unhealthyContainers = new List(); @@ -423,7 +428,7 @@ private async Task RemoveContainerNetworkAsync(IExecutionContext executionContex } } - public async Task Healthcheck(IExecutionContext executionContext, ContainerInfo container) + private async Task Healthcheck(IExecutionContext executionContext, ContainerInfo container) { string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\""; string serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); @@ -444,7 +449,7 @@ public async Task Healthcheck(IExecutionContext executionContext, Contai return serviceHealth; } - public async Task ContainerErrorLogs(IExecutionContext executionContext, ContainerInfo container) + private async Task ContainerErrorLogs(IExecutionContext executionContext, ContainerInfo container) { await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId); executionContext.Error($"Failed to initialize container {container.ContainerImage}"); diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index 7bd5675445c..032bc335165 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -6,6 +6,8 @@ using System.Threading.Tasks; using System.Collections.Generic; using System.Runtime.CompilerServices; +using GitHub.DistributedTask.WebApi; +using System; namespace GitHub.Runner.Common.Tests.Worker { @@ -18,37 +20,69 @@ public sealed class ContainerOperationProviderL0 private Mock _dockerManager; private Mock _containerHookManager; private ContainerOperationProvider containerOperationProvider; - private Mock serverQueue; - private Mock pagingLogger; - private List healthyDockerStatus = new List { "healthy" }; + private List unhealthyDockerStatus = new List { "unhealthy" }; private List dockerLogs = new List { "log1", "log2", "log3" }; - private ContainerInfo containerInfo; + List containers = new List(); [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void Healthchecktest_healthyDocker() + public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertExceptionThrown() { //Arrange Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(healthyDockerStatus)); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); + + //Act and Assert + await Assert.ThrowsAsync(() => containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers)); + + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertFailedTask() + { + //Arrange + Setup(); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); //Act - var result = await containerOperationProvider.Healthcheck(_ec.Object, containerInfo); + try + { + await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); + + } + catch (InvalidOperationException) { } //Assert - _dockerManager.Verify(dm => dm.DockerLogs(It.IsAny(), It.IsAny()), Times.Never()); + Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed); } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void RunServiceContainersHealthcheck_healthyServiceContainer_AssertSucceededTask() + { + //Arrange + Setup(); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(healthyDockerStatus)); + + //Act + await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); + //Assert + Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); + + } private void Setup([CallerMemberName] string testName = "") { - containerInfo = new ContainerInfo() { ContainerImage = "ubuntu:16.04" }; + containers.Add(new ContainerInfo() { ContainerImage = "ubuntu:16.04" }); _hc = new TestHostContext(this, "name"); _ec = new Mock(); serverQueue = new Mock(); From e03ad87776e50cc18058e409d9f651f6269936dc Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Wed, 14 Sep 2022 08:40:51 +0000 Subject: [PATCH 25/41] Remove the default value for bool variable --- src/Runner.Worker/Container/ContainerInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Worker/Container/ContainerInfo.cs b/src/Runner.Worker/Container/ContainerInfo.cs index ce883cc1096..32e55eb3c40 100644 --- a/src/Runner.Worker/Container/ContainerInfo.cs +++ b/src/Runner.Worker/Container/ContainerInfo.cs @@ -92,7 +92,7 @@ public ContainerInfo(IHostContext hostContext, Pipelines.JobContainer container, public bool IsJobContainer { get; set; } public bool IsAlpine { get; set; } - public bool FailedInitialization { get; set; } = false; + public bool FailedInitialization { get; set; } public IDictionary ContainerEnvironmentVariables { From 5b26a1e1ae2d7e7d2c93d708ee847b7faba3f8a2 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Thu, 15 Sep 2022 09:31:09 +0200 Subject: [PATCH 26/41] Update src/Runner.Worker/ContainerOperationProvider.cs Co-authored-by: Tingluo Huang --- src/Runner.Worker/ContainerOperationProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index b23ae388662..29ac5177fac 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -428,7 +428,7 @@ private async Task RemoveContainerNetworkAsync(IExecutionContext executionContex } } - private async Task Healthcheck(IExecutionContext executionContext, ContainerInfo container) + private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container) { string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\""; string serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); From a9fa7f83e9564c6a5b8a0f40ee0fa4c141c1f87f Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Thu, 15 Sep 2022 09:31:37 +0200 Subject: [PATCH 27/41] Update src/Runner.Worker/ContainerOperationProvider.cs Co-authored-by: Tingluo Huang --- src/Runner.Worker/ContainerOperationProvider.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 29ac5177fac..8433f59e5b5 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -89,7 +89,6 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec executionContext.Output("##[group]Create local container network"); var containerNetwork = $"github_network_{Guid.NewGuid().ToString("N")}"; await CreateContainerNetworkAsync(executionContext, containerNetwork); - executionContext.JobContext.Container["network"] = new StringContextData(containerNetwork); executionContext.Output("##[endgroup]"); From b15525a8caa564c4f0cd8f3e5b25219a355fd61e Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Thu, 15 Sep 2022 07:58:33 +0000 Subject: [PATCH 28/41] Rename Healthcheck back to ContainerHealthcheck --- src/Runner.Worker/ContainerOperationProvider.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 8433f59e5b5..79d2190a433 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -108,7 +108,7 @@ public async Task RunContainersHealthcheck(IExecutionContext executionContext, L var unhealthyContainers = new List(); foreach (var container in containers.Where(c => !c.IsJobContainer)) { - var healthcheck = await Healthcheck(executionContext, container); + var healthcheck = await ContainerHealthcheck(executionContext, container); if (!string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)) { @@ -328,7 +328,7 @@ private async Task StopContainerAsync(IExecutionContext executionContext, Contai { if (!container.IsJobContainer && !container.FailedInitialization) { - var healthcheck = await Healthcheck(executionContext, container); + var healthcheck = await ContainerHealthcheck(executionContext, container); if (string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)) { // Print logs for service container jobs (not the "action" job itself b/c that's already logged). From 72fe5798c3264d012d33fc7c47d8994759307658 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Thu, 15 Sep 2022 18:55:37 +0000 Subject: [PATCH 29/41] Make test sequential --- src/Test/L0/Worker/ContainerOperationProviderL0.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index 032bc335165..9cf97af5121 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -12,6 +12,7 @@ namespace GitHub.Runner.Common.Tests.Worker { + [Collection("Sequential")] public sealed class ContainerOperationProviderL0 { From 7ba0916092697b936ee4a6aa40390f7fa00c7c68 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Thu, 15 Sep 2022 19:04:13 +0000 Subject: [PATCH 30/41] Unextract the container error logs method --- src/Runner.Worker/ContainerOperationProvider.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 79d2190a433..45017133c69 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -126,7 +126,9 @@ public async Task RunContainersHealthcheck(IExecutionContext executionContext, L foreach (var container in unhealthyContainers) { executionContext.Output($"##[group]Service container {container.ContainerNetworkAlias} failed."); - await ContainerErrorLogs(executionContext, container); + await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId); + executionContext.Error($"Failed to initialize container {container.ContainerImage}"); + container.FailedInitialization = true; executionContext.Output("##[endgroup]"); } throw new InvalidOperationException("One or more containers failed to start."); @@ -448,13 +450,6 @@ private async Task ContainerHealthcheck(IExecutionContext executionConte return serviceHealth; } - private async Task ContainerErrorLogs(IExecutionContext executionContext, ContainerInfo container) - { - await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId); - executionContext.Error($"Failed to initialize container {container.ContainerImage}"); - container.FailedInitialization = true; - } - private async Task ContainerRegistryLogin(IExecutionContext executionContext, ContainerInfo container) { if (string.IsNullOrEmpty(container.RegistryAuthUsername) || string.IsNullOrEmpty(container.RegistryAuthPassword)) From 805d6fdb39da2a3e949c2d3ca8a3a2c3aeec4c82 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Thu, 15 Sep 2022 19:17:22 +0000 Subject: [PATCH 31/41] remove test asserting thrown exception --- .../L0/Worker/ContainerOperationProviderL0.cs | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index 9cf97af5121..81b71b116dc 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -12,7 +12,6 @@ namespace GitHub.Runner.Common.Tests.Worker { - [Collection("Sequential")] public sealed class ContainerOperationProviderL0 { @@ -29,20 +28,6 @@ public sealed class ContainerOperationProviderL0 List containers = new List(); - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertExceptionThrown() - { - //Arrange - Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); - - //Act and Assert - await Assert.ThrowsAsync(() => containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers)); - - } - [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] @@ -58,11 +43,11 @@ public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_Asse await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); } - catch (InvalidOperationException) { } - + catch (InvalidOperationException) { + //Assert Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed); - + } } [Fact] From 3b30e5c0d572a0ca62ec213599818dd4dd28c666 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Thu, 15 Sep 2022 19:24:57 +0000 Subject: [PATCH 32/41] Add configure await --- src/Test/L0/Worker/ContainerOperationProviderL0.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index 81b71b116dc..893d861ebd7 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -60,7 +60,7 @@ public async void RunServiceContainersHealthcheck_healthyServiceContainer_Assert _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(healthyDockerStatus)); //Act - await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); + await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers).ConfigureAwait(true); //Assert Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); From d3f463b4b0ddfe013de891c5e94370a0d9af7805 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Fri, 16 Sep 2022 09:22:06 +0200 Subject: [PATCH 33/41] Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang --- src/Test/L0/Worker/ContainerOperationProviderL0.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index 893d861ebd7..8a86ed0bb99 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -41,7 +41,6 @@ public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_Asse try { await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); - } catch (InvalidOperationException) { From 75c40e3cacd57d91ad84ff3ddf395c43c4e5a083 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Fri, 16 Sep 2022 09:22:46 +0200 Subject: [PATCH 34/41] Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang --- src/Test/L0/Worker/ContainerOperationProviderL0.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index 8a86ed0bb99..a6c4248da4b 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -45,7 +45,8 @@ public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_Asse catch (InvalidOperationException) { //Assert - Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed); + //Assert + Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed); } } From b1dd7975bc5b26556da961947f8ff3d50c194434 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Fri, 16 Sep 2022 09:23:17 +0200 Subject: [PATCH 35/41] Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang --- src/Test/L0/Worker/ContainerOperationProviderL0.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index a6c4248da4b..2aaa4ade48a 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -69,7 +69,7 @@ public async void RunServiceContainersHealthcheck_healthyServiceContainer_Assert private void Setup([CallerMemberName] string testName = "") { containers.Add(new ContainerInfo() { ContainerImage = "ubuntu:16.04" }); - _hc = new TestHostContext(this, "name"); + _hc = new TestHostContext(this, testName); _ec = new Mock(); serverQueue = new Mock(); pagingLogger = new Mock(); From 82b81f26edadf0780f0da573ea8e8053f9153c98 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Fri, 16 Sep 2022 09:23:32 +0200 Subject: [PATCH 36/41] Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang --- src/Test/L0/Worker/ContainerOperationProviderL0.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index 2aaa4ade48a..0a69cadac75 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -66,6 +66,7 @@ public async void RunServiceContainersHealthcheck_healthyServiceContainer_Assert Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); } + private void Setup([CallerMemberName] string testName = "") { containers.Add(new ContainerInfo() { ContainerImage = "ubuntu:16.04" }); From c5685b7c6309bc8cd5d6d16f981c8fff97e42807 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Fri, 16 Sep 2022 09:23:44 +0200 Subject: [PATCH 37/41] Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang --- src/Test/L0/Worker/ContainerOperationProviderL0.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index 0a69cadac75..e0a0e5af562 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -42,7 +42,8 @@ public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_Asse { await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); } - catch (InvalidOperationException) { + catch (InvalidOperationException) + { //Assert //Assert From 1778f8f022a92d61de510ab56bb9f25811e260d7 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Fri, 16 Sep 2022 07:53:12 +0000 Subject: [PATCH 38/41] Add back test asserting exception --- .../L0/Worker/ContainerOperationProviderL0.cs | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index e0a0e5af562..bfc09f343ac 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -43,14 +43,27 @@ public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_Asse await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); } catch (InvalidOperationException) - { - - //Assert + { + //Assert Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed); } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertExceptionThrown() + { + //Arrange + Setup(); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); + + //Act and Assert + await Assert.ThrowsAsync(() => containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers)); + + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] @@ -61,7 +74,7 @@ public async void RunServiceContainersHealthcheck_healthyServiceContainer_Assert _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(healthyDockerStatus)); //Act - await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers).ConfigureAwait(true); + await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); //Assert Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); From fec24e834137afe2dc751a38707d0e399c10f8fb Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Fri, 16 Sep 2022 11:09:55 +0000 Subject: [PATCH 39/41] Check service exit code if there is no healtcheck configured --- .../ContainerOperationProvider.cs | 7 ++- .../L0/Worker/ContainerOperationProviderL0.cs | 61 +++++++++++++++++-- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 45017133c69..d532f9d87bf 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -435,8 +435,13 @@ private async Task ContainerHealthcheck(IExecutionContext executionConte string serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); if (string.IsNullOrEmpty(serviceHealth)) { + string exitCode = "--format=\"{{print .State.ExitCode}}\""; + string serviceExitCode = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: exitCode)).FirstOrDefault(); + + // Container has no healthcheck but didn't exit with an error code + if ("0".Equals(serviceExitCode)) return "healthy"; // Container has no HEALTHCHECK - return String.Empty; + else return "unhealthy"; } var retryCount = 0; while (string.Equals(serviceHealth, "starting", StringComparison.OrdinalIgnoreCase)) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index bfc09f343ac..8d4c2bb66e4 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -24,7 +24,9 @@ public sealed class ContainerOperationProviderL0 private Mock pagingLogger; private List healthyDockerStatus = new List { "healthy" }; private List unhealthyDockerStatus = new List { "unhealthy" }; + private List emptyDockerStatus = new List { "" }; private List dockerLogs = new List { "log1", "log2", "log3" }; + string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\""; List containers = new List(); @@ -35,7 +37,7 @@ public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_Asse { //Arrange Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), healthCheck)).Returns(Task.FromResult(unhealthyDockerStatus)); //Act try @@ -47,6 +49,7 @@ public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_Asse //Assert Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed); + _dockerManager.Verify(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny()), Times.Once()); } } @@ -57,27 +60,77 @@ public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_Asse { //Arrange Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), healthCheck)).Returns(Task.FromResult(unhealthyDockerStatus)); //Act and Assert await Assert.ThrowsAsync(() => containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers)); + _dockerManager.Verify(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny()), Times.Once()); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void RunServiceContainersHealthcheck_HealthyServiceContainer_AssertSucceededTask() + { + //Arrange + Setup(); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), healthCheck)).Returns(Task.FromResult(healthyDockerStatus)); + + //Act + await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); + + //Assert + Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); + _dockerManager.Verify(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny()), Times.Once()); } [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void RunServiceContainersHealthcheck_healthyServiceContainer_AssertSucceededTask() + public async void RunServiceContainersHealthcheck_ServiceContainerWithoutHealthcheckAndWithOkExitStatus_AssertSucceededTask() { + + string exitCode = "--format=\"{{print .State.ExitCode}}\""; //Arrange Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(healthyDockerStatus)); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), healthCheck)).Returns(Task.FromResult(emptyDockerStatus)); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), exitCode)).Returns(Task.FromResult(new List { "0" })); //Act await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); //Assert Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); + _dockerManager.Verify(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny()), Times.Exactly(2)); + + } + + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void RunServiceContainersHealthcheck_ServiceContainerWithoutHealthcheckAndWithErrorExitStatus_AssertSucceededTask() + { + + string exitCode = "--format=\"{{print .State.ExitCode}}\""; + //Arrange + Setup(); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), healthCheck)).Returns(Task.FromResult(emptyDockerStatus)); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), exitCode)).Returns(Task.FromResult(new List { "127" })); + + //Act + try + { + await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); + + } + catch (InvalidOperationException) + { + //Assert + Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed); + _dockerManager.Verify(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny()), Times.Exactly(2)); + } } From f39a18dfd20b93e247b2fa2a14f2db492a904730 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Tue, 27 Sep 2022 07:59:44 +0000 Subject: [PATCH 40/41] Remove unnecessary healthcheck for healthy service container --- src/Runner.Worker/ContainerOperationProvider.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index d532f9d87bf..ab6cbcad54c 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -330,11 +330,6 @@ private async Task StopContainerAsync(IExecutionContext executionContext, Contai { if (!container.IsJobContainer && !container.FailedInitialization) { - var healthcheck = await ContainerHealthcheck(executionContext, container); - if (string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)) - { - // Print logs for service container jobs (not the "action" job itself b/c that's already logged). - // Print them only if the service was healthy, else they were already logged in the initialize containers section. executionContext.Output($"Print service container logs: {container.ContainerDisplayName}"); int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId); @@ -342,7 +337,6 @@ private async Task StopContainerAsync(IExecutionContext executionContext, Contai { executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); } - } } executionContext.Output($"Stop and remove container: {container.ContainerDisplayName}"); From 0d782f477f8a647fcb4ec3d3e1f1e7ac681d741f Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Mon, 3 Oct 2022 13:50:35 +0000 Subject: [PATCH 41/41] Revert "Check service exit code if there is no healtcheck configured" This reverts commit fec24e834137afe2dc751a38707d0e399c10f8fb. --- .../ContainerOperationProvider.cs | 7 +-- .../L0/Worker/ContainerOperationProviderL0.cs | 61 ++----------------- 2 files changed, 5 insertions(+), 63 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index ab6cbcad54c..2fc59730888 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -429,13 +429,8 @@ private async Task ContainerHealthcheck(IExecutionContext executionConte string serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); if (string.IsNullOrEmpty(serviceHealth)) { - string exitCode = "--format=\"{{print .State.ExitCode}}\""; - string serviceExitCode = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: exitCode)).FirstOrDefault(); - - // Container has no healthcheck but didn't exit with an error code - if ("0".Equals(serviceExitCode)) return "healthy"; // Container has no HEALTHCHECK - else return "unhealthy"; + return String.Empty; } var retryCount = 0; while (string.Equals(serviceHealth, "starting", StringComparison.OrdinalIgnoreCase)) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index 8d4c2bb66e4..bfc09f343ac 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -24,9 +24,7 @@ public sealed class ContainerOperationProviderL0 private Mock pagingLogger; private List healthyDockerStatus = new List { "healthy" }; private List unhealthyDockerStatus = new List { "unhealthy" }; - private List emptyDockerStatus = new List { "" }; private List dockerLogs = new List { "log1", "log2", "log3" }; - string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\""; List containers = new List(); @@ -37,7 +35,7 @@ public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_Asse { //Arrange Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), healthCheck)).Returns(Task.FromResult(unhealthyDockerStatus)); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); //Act try @@ -49,7 +47,6 @@ public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_Asse //Assert Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed); - _dockerManager.Verify(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny()), Times.Once()); } } @@ -60,77 +57,27 @@ public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_Asse { //Arrange Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), healthCheck)).Returns(Task.FromResult(unhealthyDockerStatus)); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); //Act and Assert await Assert.ThrowsAsync(() => containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers)); - _dockerManager.Verify(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny()), Times.Once()); - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void RunServiceContainersHealthcheck_HealthyServiceContainer_AssertSucceededTask() - { - //Arrange - Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), healthCheck)).Returns(Task.FromResult(healthyDockerStatus)); - - //Act - await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); - - //Assert - Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); - _dockerManager.Verify(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny()), Times.Once()); } [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void RunServiceContainersHealthcheck_ServiceContainerWithoutHealthcheckAndWithOkExitStatus_AssertSucceededTask() + public async void RunServiceContainersHealthcheck_healthyServiceContainer_AssertSucceededTask() { - - string exitCode = "--format=\"{{print .State.ExitCode}}\""; //Arrange Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), healthCheck)).Returns(Task.FromResult(emptyDockerStatus)); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), exitCode)).Returns(Task.FromResult(new List { "0" })); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(healthyDockerStatus)); //Act await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); //Assert Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); - _dockerManager.Verify(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny()), Times.Exactly(2)); - - } - - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void RunServiceContainersHealthcheck_ServiceContainerWithoutHealthcheckAndWithErrorExitStatus_AssertSucceededTask() - { - - string exitCode = "--format=\"{{print .State.ExitCode}}\""; - //Arrange - Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), healthCheck)).Returns(Task.FromResult(emptyDockerStatus)); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), exitCode)).Returns(Task.FromResult(new List { "127" })); - - //Act - try - { - await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); - - } - catch (InvalidOperationException) - { - //Assert - Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed); - _dockerManager.Verify(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny()), Times.Exactly(2)); - } }