From 0bb298b24feb7298acd6d321f32450bd7d5823ad Mon Sep 17 00:00:00 2001 From: Francesco Renzi Date: Mon, 12 Sep 2022 09:46:30 +0100 Subject: [PATCH 1/7] Add save-state file command --- src/Runner.Common/ExtensionManager.cs | 1 + src/Runner.Worker/FileCommandManager.cs | 45 +++++++++++++++++++++++++ src/Runner.Worker/GitHubContext.cs | 1 + 3 files changed, 47 insertions(+) diff --git a/src/Runner.Common/ExtensionManager.cs b/src/Runner.Common/ExtensionManager.cs index 29f99b62834..dcfbb0f8404 100644 --- a/src/Runner.Common/ExtensionManager.cs +++ b/src/Runner.Common/ExtensionManager.cs @@ -61,6 +61,7 @@ private List LoadExtensions() where T : class, IExtension Add(extensions, "GitHub.Runner.Worker.AddPathFileCommand, Runner.Worker"); Add(extensions, "GitHub.Runner.Worker.SetEnvFileCommand, Runner.Worker"); Add(extensions, "GitHub.Runner.Worker.CreateStepSummaryCommand, Runner.Worker"); + Add(extensions, "GitHub.Runner.Worker.SaveStateFileCommand, Runner.Worker"); break; case "GitHub.Runner.Listener.Check.ICheckExtension": Add(extensions, "GitHub.Runner.Listener.Check.InternetCheck, Runner.Listener"); diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index a55050de13b..c29d0606db6 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -325,4 +325,49 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container } } } + + public sealed class SaveStateFileCommand : RunnerService, IFileCommandExtension + { + public string ContextName => "state"; + public string FilePrefix => "save_state_"; + + public Type ExtensionType => typeof(IFileCommandExtension); + + public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) + { + if (File.Exists(filePath)) + { + var lines = File.ReadAllLines(filePath, Encoding.UTF8); + foreach(var line in lines) + { + if (string.IsNullOrEmpty(line)) + { + continue; + } + + var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); + var name = split[0]; + var value = split[1]; + + // Embedded steps (composite) keep track of the state at the root level + if (context.IsEmbedded) + { + var id = context.EmbeddedId; + if (!context.Root.EmbeddedIntraActionState.ContainsKey(id)) + { + context.Root.EmbeddedIntraActionState[id] = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + context.Root.EmbeddedIntraActionState[id][name] = value; + } + // Otherwise modify the ExecutionContext + else + { + context.IntraActionState[name] = value; + } + + context.Debug($"Save intra-action state {name} = {value}"); + } + } + } + } } diff --git a/src/Runner.Worker/GitHubContext.cs b/src/Runner.Worker/GitHubContext.cs index cdefd45b405..bb7b6f33b10 100644 --- a/src/Runner.Worker/GitHubContext.cs +++ b/src/Runner.Worker/GitHubContext.cs @@ -34,6 +34,7 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa "run_number", "server_url", "sha", + "state", "step_summary", "triggering_actor", "workflow", From 7490f6e4ade4ee2b49951812eda92bde8d935b13 Mon Sep 17 00:00:00 2001 From: Francesco Renzi Date: Mon, 12 Sep 2022 13:34:26 +0100 Subject: [PATCH 2/7] Add set-output file command --- src/Runner.Common/ExtensionManager.cs | 1 + src/Runner.Worker/FileCommandManager.cs | 31 +++++++++++++++++++++++++ src/Runner.Worker/GitHubContext.cs | 1 + 3 files changed, 33 insertions(+) diff --git a/src/Runner.Common/ExtensionManager.cs b/src/Runner.Common/ExtensionManager.cs index dcfbb0f8404..487f10ebd4a 100644 --- a/src/Runner.Common/ExtensionManager.cs +++ b/src/Runner.Common/ExtensionManager.cs @@ -62,6 +62,7 @@ private List LoadExtensions() where T : class, IExtension Add(extensions, "GitHub.Runner.Worker.SetEnvFileCommand, Runner.Worker"); Add(extensions, "GitHub.Runner.Worker.CreateStepSummaryCommand, Runner.Worker"); Add(extensions, "GitHub.Runner.Worker.SaveStateFileCommand, Runner.Worker"); + Add(extensions, "GitHub.Runner.Worker.SetOutputFileCommand, Runner.Worker"); break; case "GitHub.Runner.Listener.Check.ICheckExtension": Add(extensions, "GitHub.Runner.Listener.Check.InternetCheck, Runner.Listener"); diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index c29d0606db6..e89b02172ad 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -370,4 +370,35 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container } } } + + public sealed class SetOutputFileCommand : RunnerService, IFileCommandExtension + { + public string ContextName => "output"; + public string FilePrefix => "set_output_"; + + public Type ExtensionType => typeof(IFileCommandExtension); + + public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) + { + if (File.Exists(filePath)) + { + var lines = File.ReadAllLines(filePath, Encoding.UTF8); + foreach(var line in lines) + { + if (string.IsNullOrEmpty(line)) + { + continue; + } + + var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); + var name = split[0]; + var value = split[1]; + + context.SetOutput(name, value, out var reference); + + context.Debug($"Set output {name} = {value}"); + } + } + } + } } diff --git a/src/Runner.Worker/GitHubContext.cs b/src/Runner.Worker/GitHubContext.cs index bb7b6f33b10..f320705eba0 100644 --- a/src/Runner.Worker/GitHubContext.cs +++ b/src/Runner.Worker/GitHubContext.cs @@ -21,6 +21,7 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa "graphql_url", "head_ref", "job", + "output", "path", "ref_name", "ref_protected", From 0ac3c43500476ba517ff334cd0c485b5fb4ddbcf Mon Sep 17 00:00:00 2001 From: Francesco Renzi Date: Mon, 12 Sep 2022 17:27:24 +0100 Subject: [PATCH 3/7] Add support for heredoc syntax to save-state file command --- src/Runner.Worker/FileCommandManager.cs | 148 ++++++- src/Test/L0/Worker/SaveStateFileCommandL0.cs | 438 +++++++++++++++++++ 2 files changed, 562 insertions(+), 24 deletions(-) create mode 100644 src/Test/L0/Worker/SaveStateFileCommandL0.cs diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index e89b02172ad..ab9c36f9b4b 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -335,39 +335,139 @@ public sealed class SaveStateFileCommand : RunnerService, IFileCommandExtension public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) { - if (File.Exists(filePath)) + try { - var lines = File.ReadAllLines(filePath, Encoding.UTF8); - foreach(var line in lines) + var text = File.ReadAllText(filePath) ?? string.Empty; + var index = 0; + var line = ReadLine(text, ref index); + while (line != null) { - if (string.IsNullOrEmpty(line)) + if (!string.IsNullOrEmpty(line)) { - continue; - } + var stateName = string.Empty; + var stateValue = string.Empty; - var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); - var name = split[0]; - var value = split[1]; + var equalsIndex = line.IndexOf("=", StringComparison.Ordinal); + var heredocIndex = line.IndexOf("<<", StringComparison.Ordinal); - // Embedded steps (composite) keep track of the state at the root level - if (context.IsEmbedded) - { - var id = context.EmbeddedId; - if (!context.Root.EmbeddedIntraActionState.ContainsKey(id)) - { - context.Root.EmbeddedIntraActionState[id] = new Dictionary(StringComparer.OrdinalIgnoreCase); - } - context.Root.EmbeddedIntraActionState[id][name] = value; - } - // Otherwise modify the ExecutionContext - else - { - context.IntraActionState[name] = value; + // Normal style NAME=VALUE + if (equalsIndex >= 0 && (heredocIndex < 0 || equalsIndex < heredocIndex)) + { + var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); + if (string.IsNullOrEmpty(line)) + { + throw new Exception($"Invalid state format '{line}'. State name must not be empty"); + } + + stateName = split[0]; + stateValue = split[1]; + } + // Heredoc style NAME<= 0 && (equalsIndex < 0 || heredocIndex < equalsIndex)) + { + var split = line.Split(new[] { "<<" }, 2, StringSplitOptions.None); + if (string.IsNullOrEmpty(split[0]) || string.IsNullOrEmpty(split[1])) + { + throw new Exception($"Invalid state format '{line}'. State name must not be empty and delimiter must not be empty"); + } + stateName = split[0]; + var delimiter = split[1]; + var startIndex = index; // Start index of the value (inclusive) + var endIndex = index; // End index of the value (exclusive) + var tempLine = ReadLine(text, ref index, out var newline); + while (!string.Equals(tempLine, delimiter, StringComparison.Ordinal)) + { + if (tempLine == null) + { + throw new Exception($"Invalid state value. Matching delimiter not found '{delimiter}'"); + } + if (newline == null) + { + throw new Exception($"Invalid state value. EOF marker missing new line."); + } + endIndex = index - newline.Length; + tempLine = ReadLine(text, ref index, out newline); + } + + stateValue = endIndex > startIndex ? text.Substring(startIndex, endIndex - startIndex) : string.Empty; + } + else + { + throw new Exception($"Invalid state format '{line}'"); + } + + // Embedded steps (composite) keep track of the state at the root level + if (context.IsEmbedded) + { + var id = context.EmbeddedId; + if (!context.Root.EmbeddedIntraActionState.ContainsKey(id)) + { + context.Root.EmbeddedIntraActionState[id] = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + context.Root.EmbeddedIntraActionState[id][stateName] = stateValue; + } + // Otherwise modify the ExecutionContext + else + { + context.IntraActionState[stateName] = stateValue; + } + + context.Debug($"Save intra-action state {stateName} = {stateValue}"); } - context.Debug($"Save intra-action state {name} = {value}"); + line = ReadLine(text, ref index); } } + catch (DirectoryNotFoundException) + { + context.Debug($"State file does not exist '{filePath}'"); + } + catch (FileNotFoundException) + { + context.Debug($"State file does not exist '{filePath}'"); + } + } + + private static string ReadLine( + string text, + ref int index) + { + return ReadLine(text, ref index, out _); + } + + private static string ReadLine( + string text, + ref int index, + out string newline) + { + if (index >= text.Length) + { + newline = null; + return null; + } + + var originalIndex = index; + var lfIndex = text.IndexOf("\n", index, StringComparison.Ordinal); + if (lfIndex < 0) + { + index = text.Length; + newline = null; + return text.Substring(originalIndex); + } + +#if OS_WINDOWS + var crLFIndex = text.IndexOf("\r\n", index, StringComparison.Ordinal); + if (crLFIndex >= 0 && crLFIndex < lfIndex) + { + index = crLFIndex + 2; // Skip over CRLF + newline = "\r\n"; + return text.Substring(originalIndex, crLFIndex - originalIndex); + } +#endif + + index = lfIndex + 1; // Skip over LF + newline = "\n"; + return text.Substring(originalIndex, lfIndex - originalIndex); } } diff --git a/src/Test/L0/Worker/SaveStateFileCommandL0.cs b/src/Test/L0/Worker/SaveStateFileCommandL0.cs new file mode 100644 index 00000000000..45296c70978 --- /dev/null +++ b/src/Test/L0/Worker/SaveStateFileCommandL0.cs @@ -0,0 +1,438 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Runtime.CompilerServices; +using GitHub.Runner.Common.Util; +using GitHub.Runner.Sdk; +using GitHub.Runner.Worker; +using GitHub.Runner.Worker.Container; +using GitHub.Runner.Worker.Handlers; +using Moq; +using Xunit; +using DTWebApi = GitHub.DistributedTask.WebApi; + +namespace GitHub.Runner.Common.Tests.Worker +{ + public sealed class SaveStateFileCommandL0 + { + private Mock _executionContext; + private List> _issues; + private string _rootDirectory; + private SaveStateFileCommand _saveStateFileCommand; + private Dictionary _intraActionState; + private ITraceWriter _trace; + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_DirectoryNotFound() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "directory-not-found", "env"); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _intraActionState.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_NotFound() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "file-not-found"); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _intraActionState.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_EmptyFile() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "empty-file"); + var content = new List(); + WriteContent(stateFile, content); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _intraActionState.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Simple() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_STATE=MY VALUE", + }; + WriteContent(stateFile, content); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(1, _intraActionState.Count); + Assert.Equal("MY VALUE", _intraActionState["MY_STATE"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Simple_SkipEmptyLines() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + string.Empty, + "MY_STATE=my value", + string.Empty, + "MY_STATE_2=my second value", + string.Empty, + }; + WriteContent(stateFile, content); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(2, _intraActionState.Count); + Assert.Equal("my value", _intraActionState["MY_STATE"]); + Assert.Equal("my second value", _intraActionState["MY_STATE_2"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Simple_EmptyValue() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple-empty-value"); + var content = new List + { + "MY_STATE=", + }; + WriteContent(stateFile, content); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(1, _intraActionState.Count); + Assert.Equal(string.Empty, _intraActionState["MY_STATE"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Simple_MultipleValues() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_STATE=my value", + "MY_STATE_2=", + "MY_STATE_3=my third value", + }; + WriteContent(stateFile, content); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(3, _intraActionState.Count); + Assert.Equal("my value", _intraActionState["MY_STATE"]); + Assert.Equal(string.Empty, _intraActionState["MY_STATE_2"]); + Assert.Equal("my third value", _intraActionState["MY_STATE_3"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Simple_SpecialCharacters() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_STATE==abc", + "MY_STATE_2=def=ghi", + "MY_STATE_3=jkl=", + }; + WriteContent(stateFile, content); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(3, _intraActionState.Count); + Assert.Equal("=abc", _intraActionState["MY_STATE"]); + Assert.Equal("def=ghi", _intraActionState["MY_STATE_2"]); + Assert.Equal("jkl=", _intraActionState["MY_STATE_3"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Heredoc() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_STATE< + { + "MY_STATE< + { + string.Empty, + "MY_STATE< + { + "MY_STATE<<=EOF", + "hello", + "one", + "=EOF", + "MY_STATE_2<< + { + "MY_STATE<(() => _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null)); + Assert.Contains("Matching delimiter not found", ex.Message); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Heredoc_MissingNewLineMultipleLines() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_STATE<(() => _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null)); + Assert.Contains("EOF marker missing new line", ex.Message); + } + } + +#if OS_WINDOWS + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Heredoc_PreservesNewline() + { + using (var hostContext = Setup()) + { + var newline = "\n"; + var stateFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_STATE< content, + string newline = null) + { + if (string.IsNullOrEmpty(newline)) + { + newline = Environment.NewLine; + } + + var encoding = new UTF8Encoding(true); // Emit BOM + var contentStr = string.Join(newline, content); + File.WriteAllText(path, contentStr, encoding); + } + + private TestHostContext Setup([CallerMemberName] string name = "") + { + _issues = new List>(); + _intraActionState = new Dictionary(); + + var hostContext = new TestHostContext(this, name); + + // Trace + _trace = hostContext.GetTrace(); + + // Directory for test data + var workDirectory = hostContext.GetDirectory(WellKnownDirectory.Work); + ArgUtil.NotNullOrEmpty(workDirectory, nameof(workDirectory)); + Directory.CreateDirectory(workDirectory); + _rootDirectory = Path.Combine(workDirectory, nameof(SaveStateFileCommandL0)); + Directory.CreateDirectory(_rootDirectory); + + // Execution context + _executionContext = new Mock(); + _executionContext.Setup(x => x.Global) + .Returns(new GlobalContext + { + EnvironmentVariables = new Dictionary(VarUtil.EnvironmentVariableKeyComparer), + WriteDebug = true, + }); + _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((DTWebApi.Issue issue, string logMessage) => + { + _issues.Add(new Tuple(issue, logMessage)); + var message = !string.IsNullOrEmpty(logMessage) ? logMessage : issue.Message; + _trace.Info($"Issue '{issue.Type}': {message}"); + }); + _executionContext.Setup(x => x.Write(It.IsAny(), It.IsAny())) + .Callback((string tag, string message) => + { + _trace.Info($"{tag}{message}"); + }); + _executionContext.Setup(x => x.IntraActionState) + .Returns(_intraActionState); + + // SaveStateFileCommand + _saveStateFileCommand = new SaveStateFileCommand(); + _saveStateFileCommand.Initialize(hostContext); + + return hostContext; + } + } +} From f5dc44d9fbb241d6c3a1b7c2d7a4c64bd675bc01 Mon Sep 17 00:00:00 2001 From: Francesco Renzi Date: Mon, 12 Sep 2022 17:52:43 +0100 Subject: [PATCH 4/7] Add support for heredoc syntax to set-output file command --- src/Runner.Worker/FileCommandManager.cs | 122 ++++- src/Test/L0/Worker/SetOutputFileCommandL0.cs | 444 +++++++++++++++++++ 2 files changed, 555 insertions(+), 11 deletions(-) create mode 100644 src/Test/L0/Worker/SetOutputFileCommandL0.cs diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index ab9c36f9b4b..1e7cc9d0b5e 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -480,25 +480,125 @@ public sealed class SetOutputFileCommand : RunnerService, IFileCommandExtension public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) { - if (File.Exists(filePath)) + try { - var lines = File.ReadAllLines(filePath, Encoding.UTF8); - foreach(var line in lines) + var text = File.ReadAllText(filePath) ?? string.Empty; + var index = 0; + var line = ReadLine(text, ref index); + while (line != null) { - if (string.IsNullOrEmpty(line)) + if (!string.IsNullOrEmpty(line)) { - continue; - } + var outputName = string.Empty; + var outputValue = string.Empty; + + var equalsIndex = line.IndexOf("=", StringComparison.Ordinal); + var heredocIndex = line.IndexOf("<<", StringComparison.Ordinal); + + // Normal style NAME=VALUE + if (equalsIndex >= 0 && (heredocIndex < 0 || equalsIndex < heredocIndex)) + { + var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); + if (string.IsNullOrEmpty(line)) + { + throw new Exception($"Invalid state format '{line}'. State name must not be empty"); + } + + outputName = split[0]; + outputValue = split[1]; + } + // Heredoc style NAME<= 0 && (equalsIndex < 0 || heredocIndex < equalsIndex)) + { + var split = line.Split(new[] { "<<" }, 2, StringSplitOptions.None); + if (string.IsNullOrEmpty(split[0]) || string.IsNullOrEmpty(split[1])) + { + throw new Exception($"Invalid state format '{line}'. State name must not be empty and delimiter must not be empty"); + } + outputName = split[0]; + var delimiter = split[1]; + var startIndex = index; // Start index of the value (inclusive) + var endIndex = index; // End index of the value (exclusive) + var tempLine = ReadLine(text, ref index, out var newline); + while (!string.Equals(tempLine, delimiter, StringComparison.Ordinal)) + { + if (tempLine == null) + { + throw new Exception($"Invalid state value. Matching delimiter not found '{delimiter}'"); + } + if (newline == null) + { + throw new Exception($"Invalid state value. EOF marker missing new line."); + } + endIndex = index - newline.Length; + tempLine = ReadLine(text, ref index, out newline); + } + + outputValue = endIndex > startIndex ? text.Substring(startIndex, endIndex - startIndex) : string.Empty; + } + else + { + throw new Exception($"Invalid state format '{line}'"); + } - var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); - var name = split[0]; - var value = split[1]; + context.SetOutput(outputName, outputValue, out var reference); - context.SetOutput(name, value, out var reference); + context.Debug($"Set output {outputName} = {outputValue}"); + } - context.Debug($"Set output {name} = {value}"); + line = ReadLine(text, ref index); } } + catch (DirectoryNotFoundException) + { + context.Debug($"State file does not exist '{filePath}'"); + } + catch (FileNotFoundException) + { + context.Debug($"State file does not exist '{filePath}'"); + } + } + + private static string ReadLine( + string text, + ref int index) + { + return ReadLine(text, ref index, out _); + } + + private static string ReadLine( + string text, + ref int index, + out string newline) + { + if (index >= text.Length) + { + newline = null; + return null; + } + + var originalIndex = index; + var lfIndex = text.IndexOf("\n", index, StringComparison.Ordinal); + if (lfIndex < 0) + { + index = text.Length; + newline = null; + return text.Substring(originalIndex); + } + +#if OS_WINDOWS + var crLFIndex = text.IndexOf("\r\n", index, StringComparison.Ordinal); + if (crLFIndex >= 0 && crLFIndex < lfIndex) + { + index = crLFIndex + 2; // Skip over CRLF + newline = "\r\n"; + return text.Substring(originalIndex, crLFIndex - originalIndex); + } +#endif + + index = lfIndex + 1; // Skip over LF + newline = "\n"; + return text.Substring(originalIndex, lfIndex - originalIndex); } } } diff --git a/src/Test/L0/Worker/SetOutputFileCommandL0.cs b/src/Test/L0/Worker/SetOutputFileCommandL0.cs new file mode 100644 index 00000000000..a5734c44efe --- /dev/null +++ b/src/Test/L0/Worker/SetOutputFileCommandL0.cs @@ -0,0 +1,444 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Runtime.CompilerServices; +using GitHub.Runner.Common.Util; +using GitHub.Runner.Sdk; +using GitHub.Runner.Worker; +using GitHub.Runner.Worker.Container; +using GitHub.Runner.Worker.Handlers; +using Moq; +using Xunit; +using DTWebApi = GitHub.DistributedTask.WebApi; + +namespace GitHub.Runner.Common.Tests.Worker +{ + public sealed class SetOutputFileCommandL0 + { + private Mock _executionContext; + private List> _issues; + private Dictionary _outputs; + private string _rootDirectory; + private SetOutputFileCommand _setOutputFileCommand; + private ITraceWriter _trace; + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_DirectoryNotFound() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "directory-not-found", "env"); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _outputs.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_NotFound() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "file-not-found"); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _outputs.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_EmptyFile() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "empty-file"); + var content = new List(); + WriteContent(stateFile, content); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _outputs.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Simple() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_OUTPUT=MY VALUE", + }; + WriteContent(stateFile, content); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(1, _outputs.Count); + Assert.Equal("MY VALUE", _outputs["MY_OUTPUT"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Simple_SkipEmptyLines() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + string.Empty, + "MY_OUTPUT=my value", + string.Empty, + "MY_OUTPUT_2=my second value", + string.Empty, + }; + WriteContent(stateFile, content); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(2, _outputs.Count); + Assert.Equal("my value", _outputs["MY_OUTPUT"]); + Assert.Equal("my second value", _outputs["MY_OUTPUT_2"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Simple_EmptyValue() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple-empty-value"); + var content = new List + { + "MY_OUTPUT=", + }; + WriteContent(stateFile, content); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(1, _outputs.Count); + Assert.Equal(string.Empty, _outputs["MY_OUTPUT"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Simple_MultipleValues() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_OUTPUT=my value", + "MY_OUTPUT_2=", + "MY_OUTPUT_3=my third value", + }; + WriteContent(stateFile, content); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(3, _outputs.Count); + Assert.Equal("my value", _outputs["MY_OUTPUT"]); + Assert.Equal(string.Empty, _outputs["MY_OUTPUT_2"]); + Assert.Equal("my third value", _outputs["MY_OUTPUT_3"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Simple_SpecialCharacters() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_OUTPUT==abc", + "MY_OUTPUT_2=def=ghi", + "MY_OUTPUT_3=jkl=", + }; + WriteContent(stateFile, content); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(3, _outputs.Count); + Assert.Equal("=abc", _outputs["MY_OUTPUT"]); + Assert.Equal("def=ghi", _outputs["MY_OUTPUT_2"]); + Assert.Equal("jkl=", _outputs["MY_OUTPUT_3"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Heredoc() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_OUTPUT< + { + "MY_OUTPUT< + { + string.Empty, + "MY_OUTPUT< + { + "MY_OUTPUT<<=EOF", + "hello", + "one", + "=EOF", + "MY_OUTPUT_2<< + { + "MY_OUTPUT<(() => _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null)); + Assert.Contains("Matching delimiter not found", ex.Message); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Heredoc_MissingNewLineMultipleLines() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_OUTPUT<(() => _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null)); + Assert.Contains("EOF marker missing new line", ex.Message); + } + } + +#if OS_WINDOWS + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Heredoc_PreservesNewline() + { + using (var hostContext = Setup()) + { + var newline = "\n"; + var stateFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_OUTPUT< content, + string newline = null) + { + if (string.IsNullOrEmpty(newline)) + { + newline = Environment.NewLine; + } + + var encoding = new UTF8Encoding(true); // Emit BOM + var contentStr = string.Join(newline, content); + File.WriteAllText(path, contentStr, encoding); + } + + private TestHostContext Setup([CallerMemberName] string name = "") + { + _issues = new List>(); + _outputs = new Dictionary(); + + var hostContext = new TestHostContext(this, name); + + // Trace + _trace = hostContext.GetTrace(); + + // Directory for test data + var workDirectory = hostContext.GetDirectory(WellKnownDirectory.Work); + ArgUtil.NotNullOrEmpty(workDirectory, nameof(workDirectory)); + Directory.CreateDirectory(workDirectory); + _rootDirectory = Path.Combine(workDirectory, nameof(SaveStateFileCommandL0)); + Directory.CreateDirectory(_rootDirectory); + + // Execution context + _executionContext = new Mock(); + _executionContext.Setup(x => x.Global) + .Returns(new GlobalContext + { + EnvironmentVariables = new Dictionary(VarUtil.EnvironmentVariableKeyComparer), + WriteDebug = true, + }); + _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((DTWebApi.Issue issue, string logMessage) => + { + _issues.Add(new Tuple(issue, logMessage)); + var message = !string.IsNullOrEmpty(logMessage) ? logMessage : issue.Message; + _trace.Info($"Issue '{issue.Type}': {message}"); + }); + _executionContext.Setup(x => x.Write(It.IsAny(), It.IsAny())) + .Callback((string tag, string message) => + { + _trace.Info($"{tag}{message}"); + }); + + var reference = string.Empty; + _executionContext.Setup(x => x.SetOutput(It.IsAny(), It.IsAny(), out reference)) + .Callback((string name, string value, out string reference) => + { + reference = value; + _outputs[name] = value; + }); + + // SaveStateFileCommand + _setOutputFileCommand = new SetOutputFileCommand(); + _setOutputFileCommand.Initialize(hostContext); + + return hostContext; + } + } +} From f7652c94843099b0a5773cdc693612b6fc6575d0 Mon Sep 17 00:00:00 2001 From: Francesco Renzi Date: Mon, 12 Sep 2022 18:01:08 +0100 Subject: [PATCH 5/7] Fix copy pasta mistakes in tests --- src/Test/L0/Worker/SetOutputFileCommandL0.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Test/L0/Worker/SetOutputFileCommandL0.cs b/src/Test/L0/Worker/SetOutputFileCommandL0.cs index a5734c44efe..8af9695db46 100644 --- a/src/Test/L0/Worker/SetOutputFileCommandL0.cs +++ b/src/Test/L0/Worker/SetOutputFileCommandL0.cs @@ -365,7 +365,7 @@ public void SetOutputFileCommand_Heredoc_PreservesNewline() "EOF", }; WriteContent(stateFile, content, newline: newline); - _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); Assert.Equal(0, _issues.Count); Assert.Equal(1, _outputs.Count); Assert.Equal($"hello{newline}world", _outputs["MY_OUTPUT"]); @@ -402,7 +402,7 @@ private TestHostContext Setup([CallerMemberName] string name = "") var workDirectory = hostContext.GetDirectory(WellKnownDirectory.Work); ArgUtil.NotNullOrEmpty(workDirectory, nameof(workDirectory)); Directory.CreateDirectory(workDirectory); - _rootDirectory = Path.Combine(workDirectory, nameof(SaveStateFileCommandL0)); + _rootDirectory = Path.Combine(workDirectory, nameof(SetOutputFileCommandL0)); Directory.CreateDirectory(_rootDirectory); // Execution context @@ -434,7 +434,7 @@ private TestHostContext Setup([CallerMemberName] string name = "") _outputs[name] = value; }); - // SaveStateFileCommand + // SetOutputFileCommand _setOutputFileCommand = new SetOutputFileCommand(); _setOutputFileCommand.Initialize(hostContext); From 55d63f194b9a42d5fe8cee1b56635e6dd7ef061f Mon Sep 17 00:00:00 2001 From: Francesco Renzi Date: Tue, 13 Sep 2022 11:03:05 +0100 Subject: [PATCH 6/7] Refactor reading env files --- src/Runner.Worker/FileCommandManager.cs | 339 ++++++++---------------- 1 file changed, 106 insertions(+), 233 deletions(-) diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index 1e7cc9d0b5e..ad3e3e3f29a 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -142,63 +142,10 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container { try { - var text = File.ReadAllText(filePath) ?? string.Empty; - var index = 0; - var line = ReadLine(text, ref index); - while (line != null) + var pairs = new EnvFileKeyValuePairs(filePath); + foreach (var pair in pairs) { - if (!string.IsNullOrEmpty(line)) - { - var equalsIndex = line.IndexOf("=", StringComparison.Ordinal); - var heredocIndex = line.IndexOf("<<", StringComparison.Ordinal); - - // Normal style NAME=VALUE - if (equalsIndex >= 0 && (heredocIndex < 0 || equalsIndex < heredocIndex)) - { - var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); - if (string.IsNullOrEmpty(line)) - { - throw new Exception($"Invalid environment variable format '{line}'. Environment variable name must not be empty"); - } - SetEnvironmentVariable(context, split[0], split[1]); - } - // Heredoc style NAME<= 0 && (equalsIndex < 0 || heredocIndex < equalsIndex)) - { - var split = line.Split(new[] { "<<" }, 2, StringSplitOptions.None); - if (string.IsNullOrEmpty(split[0]) || string.IsNullOrEmpty(split[1])) - { - throw new Exception($"Invalid environment variable format '{line}'. Environment variable name must not be empty and delimiter must not be empty"); - } - var name = split[0]; - var delimiter = split[1]; - var startIndex = index; // Start index of the value (inclusive) - var endIndex = index; // End index of the value (exclusive) - var tempLine = ReadLine(text, ref index, out var newline); - while (!string.Equals(tempLine, delimiter, StringComparison.Ordinal)) - { - if (tempLine == null) - { - throw new Exception($"Invalid environment variable value. Matching delimiter not found '{delimiter}'"); - } - if (newline == null) - { - throw new Exception($"Invalid environment variable value. EOF marker missing new line."); - } - endIndex = index - newline.Length; - tempLine = ReadLine(text, ref index, out newline); - } - - var value = endIndex > startIndex ? text.Substring(startIndex, endIndex - startIndex) : string.Empty; - SetEnvironmentVariable(context, name, value); - } - else - { - throw new Exception($"Invalid environment variable format '{line}'"); - } - } - - line = ReadLine(text, ref index); + SetEnvironmentVariable(context, pair.Key, pair.Value); } } catch (DirectoryNotFoundException) @@ -220,48 +167,6 @@ private static void SetEnvironmentVariable( context.SetEnvContext(name, value); context.Debug($"{name}='{value}'"); } - - private static string ReadLine( - string text, - ref int index) - { - return ReadLine(text, ref index, out _); - } - - private static string ReadLine( - string text, - ref int index, - out string newline) - { - if (index >= text.Length) - { - newline = null; - return null; - } - - var originalIndex = index; - var lfIndex = text.IndexOf("\n", index, StringComparison.Ordinal); - if (lfIndex < 0) - { - index = text.Length; - newline = null; - return text.Substring(originalIndex); - } - -#if OS_WINDOWS - var crLFIndex = text.IndexOf("\r\n", index, StringComparison.Ordinal); - if (crLFIndex >= 0 && crLFIndex < lfIndex) - { - index = crLFIndex + 2; // Skip over CRLF - newline = "\r\n"; - return text.Substring(originalIndex, crLFIndex - originalIndex); - } -#endif - - index = lfIndex + 1; // Skip over LF - newline = "\n"; - return text.Substring(originalIndex, lfIndex - originalIndex); - } } public sealed class CreateStepSummaryCommand : RunnerService, IFileCommandExtension @@ -337,85 +242,26 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container { try { - var text = File.ReadAllText(filePath) ?? string.Empty; - var index = 0; - var line = ReadLine(text, ref index); - while (line != null) + var pairs = new EnvFileKeyValuePairs(filePath); + foreach (var pair in pairs) { - if (!string.IsNullOrEmpty(line)) + // Embedded steps (composite) keep track of the state at the root level + if (context.IsEmbedded) { - var stateName = string.Empty; - var stateValue = string.Empty; - - var equalsIndex = line.IndexOf("=", StringComparison.Ordinal); - var heredocIndex = line.IndexOf("<<", StringComparison.Ordinal); - - // Normal style NAME=VALUE - if (equalsIndex >= 0 && (heredocIndex < 0 || equalsIndex < heredocIndex)) - { - var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); - if (string.IsNullOrEmpty(line)) - { - throw new Exception($"Invalid state format '{line}'. State name must not be empty"); - } - - stateName = split[0]; - stateValue = split[1]; - } - // Heredoc style NAME<= 0 && (equalsIndex < 0 || heredocIndex < equalsIndex)) - { - var split = line.Split(new[] { "<<" }, 2, StringSplitOptions.None); - if (string.IsNullOrEmpty(split[0]) || string.IsNullOrEmpty(split[1])) - { - throw new Exception($"Invalid state format '{line}'. State name must not be empty and delimiter must not be empty"); - } - stateName = split[0]; - var delimiter = split[1]; - var startIndex = index; // Start index of the value (inclusive) - var endIndex = index; // End index of the value (exclusive) - var tempLine = ReadLine(text, ref index, out var newline); - while (!string.Equals(tempLine, delimiter, StringComparison.Ordinal)) - { - if (tempLine == null) - { - throw new Exception($"Invalid state value. Matching delimiter not found '{delimiter}'"); - } - if (newline == null) - { - throw new Exception($"Invalid state value. EOF marker missing new line."); - } - endIndex = index - newline.Length; - tempLine = ReadLine(text, ref index, out newline); - } - - stateValue = endIndex > startIndex ? text.Substring(startIndex, endIndex - startIndex) : string.Empty; - } - else - { - throw new Exception($"Invalid state format '{line}'"); - } - - // Embedded steps (composite) keep track of the state at the root level - if (context.IsEmbedded) + var id = context.EmbeddedId; + if (!context.Root.EmbeddedIntraActionState.ContainsKey(id)) { - var id = context.EmbeddedId; - if (!context.Root.EmbeddedIntraActionState.ContainsKey(id)) - { context.Root.EmbeddedIntraActionState[id] = new Dictionary(StringComparer.OrdinalIgnoreCase); - } - context.Root.EmbeddedIntraActionState[id][stateName] = stateValue; } - // Otherwise modify the ExecutionContext - else - { - context.IntraActionState[stateName] = stateValue; - } - - context.Debug($"Save intra-action state {stateName} = {stateValue}"); + context.Root.EmbeddedIntraActionState[id][pair.Key] = pair.Value; + } + // Otherwise modify the ExecutionContext + else + { + context.IntraActionState[pair.Key] = pair.Value; } - line = ReadLine(text, ref index); + context.Debug($"Save intra-action state {pair.Key} = {pair.Value}"); } } catch (DirectoryNotFoundException) @@ -482,94 +328,120 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container { try { - var text = File.ReadAllText(filePath) ?? string.Empty; - var index = 0; - var line = ReadLine(text, ref index); - while (line != null) + var pairs = new EnvFileKeyValuePairs(filePath); + foreach (var pair in pairs) { - if (!string.IsNullOrEmpty(line)) - { - var outputName = string.Empty; - var outputValue = string.Empty; + context.SetOutput(pair.Key, pair.Value, out var reference); + context.Debug($"Set output {pair.Key} = {pair.Value}"); + } + } + catch (DirectoryNotFoundException) + { + context.Debug($"State file does not exist '{filePath}'"); + } + catch (FileNotFoundException) + { + context.Debug($"State file does not exist '{filePath}'"); + } + } + } + + public sealed class EnvFileKeyValuePairs: IEnumerable> + { + private string _filePath; - var equalsIndex = line.IndexOf("=", StringComparison.Ordinal); - var heredocIndex = line.IndexOf("<<", StringComparison.Ordinal); + public EnvFileKeyValuePairs(string filePath) + { + _filePath = filePath; + } - // Normal style NAME=VALUE - if (equalsIndex >= 0 && (heredocIndex < 0 || equalsIndex < heredocIndex)) + public IEnumerator> GetEnumerator() + { + var text = File.ReadAllText(_filePath) ?? string.Empty; + var index = 0; + var line = ReadLine(text, ref index); + while (line != null) + { + if (!string.IsNullOrEmpty(line)) + { + var key = string.Empty; + var output = string.Empty; + + var equalsIndex = line.IndexOf("=", StringComparison.Ordinal); + var heredocIndex = line.IndexOf("<<", StringComparison.Ordinal); + + // Normal style NAME=VALUE + if (equalsIndex >= 0 && (heredocIndex < 0 || equalsIndex < heredocIndex)) + { + var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); + if (string.IsNullOrEmpty(line)) { - var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); - if (string.IsNullOrEmpty(line)) - { - throw new Exception($"Invalid state format '{line}'. State name must not be empty"); - } + throw new Exception($"Invalid format '{line}'. Name must not be empty"); + } + + key = split[0]; + output = split[1]; + } - outputName = split[0]; - outputValue = split[1]; + // Heredoc style NAME<= 0 && (equalsIndex < 0 || heredocIndex < equalsIndex)) + { + var split = line.Split(new[] { "<<" }, 2, StringSplitOptions.None); + if (string.IsNullOrEmpty(split[0]) || string.IsNullOrEmpty(split[1])) + { + throw new Exception($"Invalid format '{line}'. Name must not be empty and delimiter must not be empty"); } - // Heredoc style NAME<= 0 && (equalsIndex < 0 || heredocIndex < equalsIndex)) + key = split[0]; + var delimiter = split[1]; + var startIndex = index; // Start index of the value (inclusive) + var endIndex = index; // End index of the value (exclusive) + var tempLine = ReadLine(text, ref index, out var newline); + while (!string.Equals(tempLine, delimiter, StringComparison.Ordinal)) { - var split = line.Split(new[] { "<<" }, 2, StringSplitOptions.None); - if (string.IsNullOrEmpty(split[0]) || string.IsNullOrEmpty(split[1])) + if (tempLine == null) { - throw new Exception($"Invalid state format '{line}'. State name must not be empty and delimiter must not be empty"); + throw new Exception($"Invalid value. Matching delimiter not found '{delimiter}'"); } - outputName = split[0]; - var delimiter = split[1]; - var startIndex = index; // Start index of the value (inclusive) - var endIndex = index; // End index of the value (exclusive) - var tempLine = ReadLine(text, ref index, out var newline); - while (!string.Equals(tempLine, delimiter, StringComparison.Ordinal)) + if (newline == null) { - if (tempLine == null) - { - throw new Exception($"Invalid state value. Matching delimiter not found '{delimiter}'"); - } - if (newline == null) - { - throw new Exception($"Invalid state value. EOF marker missing new line."); - } - endIndex = index - newline.Length; - tempLine = ReadLine(text, ref index, out newline); + throw new Exception($"Invalid value. EOF marker missing new line."); } - - outputValue = endIndex > startIndex ? text.Substring(startIndex, endIndex - startIndex) : string.Empty; - } - else - { - throw new Exception($"Invalid state format '{line}'"); + endIndex = index - newline.Length; + tempLine = ReadLine(text, ref index, out newline); } - context.SetOutput(outputName, outputValue, out var reference); - - context.Debug($"Set output {outputName} = {outputValue}"); + output = endIndex > startIndex ? text.Substring(startIndex, endIndex - startIndex) : string.Empty; + } + else + { + throw new Exception($"Invalid format '{line}'"); } - line = ReadLine(text, ref index); + yield return new KeyValuePair(key, output); } - } - catch (DirectoryNotFoundException) - { - context.Debug($"State file does not exist '{filePath}'"); - } - catch (FileNotFoundException) - { - context.Debug($"State file does not exist '{filePath}'"); + + line = ReadLine(text, ref index); } } + System.Collections.IEnumerator + System.Collections.IEnumerable.GetEnumerator() + { + // Invoke IEnumerator GetEnumerator() above. + return GetEnumerator(); + } + private static string ReadLine( - string text, - ref int index) + string text, + ref int index) { return ReadLine(text, ref index, out _); } private static string ReadLine( - string text, - ref int index, - out string newline) + string text, + ref int index, + out string newline) { if (index >= text.Length) { @@ -598,6 +470,7 @@ private static string ReadLine( index = lfIndex + 1; // Skip over LF newline = "\n"; + var reader = new EnvFileKeyValuePairs("path"); return text.Substring(originalIndex, lfIndex - originalIndex); } } From 67d66d70099d700fc5bc13d1318a6aef539de07d Mon Sep 17 00:00:00 2001 From: Francesco Renzi Date: Wed, 14 Sep 2022 10:28:38 +0100 Subject: [PATCH 7/7] Move try/catch into EnvfileKeyValuePairs --- src/Runner.Worker/FileCommandManager.cs | 141 +++++++----------------- 1 file changed, 41 insertions(+), 100 deletions(-) diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index ad3e3e3f29a..d5947e64cfe 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -140,21 +140,10 @@ public sealed class SetEnvFileCommand : RunnerService, IFileCommandExtension public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) { - try - { - var pairs = new EnvFileKeyValuePairs(filePath); - foreach (var pair in pairs) - { - SetEnvironmentVariable(context, pair.Key, pair.Value); - } - } - catch (DirectoryNotFoundException) + var pairs = new EnvFileKeyValuePairs(context, filePath); + foreach (var pair in pairs) { - context.Debug($"Environment variables file does not exist '{filePath}'"); - } - catch (FileNotFoundException) - { - context.Debug($"Environment variables file does not exist '{filePath}'"); + SetEnvironmentVariable(context, pair.Key, pair.Value); } } @@ -240,80 +229,27 @@ public sealed class SaveStateFileCommand : RunnerService, IFileCommandExtension public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) { - try + var pairs = new EnvFileKeyValuePairs(context, filePath); + foreach (var pair in pairs) { - var pairs = new EnvFileKeyValuePairs(filePath); - foreach (var pair in pairs) + // Embedded steps (composite) keep track of the state at the root level + if (context.IsEmbedded) { - // Embedded steps (composite) keep track of the state at the root level - if (context.IsEmbedded) + var id = context.EmbeddedId; + if (!context.Root.EmbeddedIntraActionState.ContainsKey(id)) { - var id = context.EmbeddedId; - if (!context.Root.EmbeddedIntraActionState.ContainsKey(id)) - { - context.Root.EmbeddedIntraActionState[id] = new Dictionary(StringComparer.OrdinalIgnoreCase); - } - context.Root.EmbeddedIntraActionState[id][pair.Key] = pair.Value; - } - // Otherwise modify the ExecutionContext - else - { - context.IntraActionState[pair.Key] = pair.Value; + context.Root.EmbeddedIntraActionState[id] = new Dictionary(StringComparer.OrdinalIgnoreCase); } - - context.Debug($"Save intra-action state {pair.Key} = {pair.Value}"); + context.Root.EmbeddedIntraActionState[id][pair.Key] = pair.Value; + } + // Otherwise modify the ExecutionContext + else + { + context.IntraActionState[pair.Key] = pair.Value; } - } - catch (DirectoryNotFoundException) - { - context.Debug($"State file does not exist '{filePath}'"); - } - catch (FileNotFoundException) - { - context.Debug($"State file does not exist '{filePath}'"); - } - } - - private static string ReadLine( - string text, - ref int index) - { - return ReadLine(text, ref index, out _); - } - - private static string ReadLine( - string text, - ref int index, - out string newline) - { - if (index >= text.Length) - { - newline = null; - return null; - } - - var originalIndex = index; - var lfIndex = text.IndexOf("\n", index, StringComparison.Ordinal); - if (lfIndex < 0) - { - index = text.Length; - newline = null; - return text.Substring(originalIndex); - } -#if OS_WINDOWS - var crLFIndex = text.IndexOf("\r\n", index, StringComparison.Ordinal); - if (crLFIndex >= 0 && crLFIndex < lfIndex) - { - index = crLFIndex + 2; // Skip over CRLF - newline = "\r\n"; - return text.Substring(originalIndex, crLFIndex - originalIndex); + context.Debug($"Save intra-action state {pair.Key} = {pair.Value}"); } -#endif - - index = lfIndex + 1; // Skip over LF - newline = "\n"; - return text.Substring(originalIndex, lfIndex - originalIndex); } } @@ -326,38 +262,44 @@ public sealed class SetOutputFileCommand : RunnerService, IFileCommandExtension public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) { - try - { - var pairs = new EnvFileKeyValuePairs(filePath); - foreach (var pair in pairs) - { - context.SetOutput(pair.Key, pair.Value, out var reference); - context.Debug($"Set output {pair.Key} = {pair.Value}"); - } - } - catch (DirectoryNotFoundException) + var pairs = new EnvFileKeyValuePairs(context, filePath); + foreach (var pair in pairs) { - context.Debug($"State file does not exist '{filePath}'"); - } - catch (FileNotFoundException) - { - context.Debug($"State file does not exist '{filePath}'"); + context.SetOutput(pair.Key, pair.Value, out var reference); + context.Debug($"Set output {pair.Key} = {pair.Value}"); } } } public sealed class EnvFileKeyValuePairs: IEnumerable> { + private IExecutionContext _context; private string _filePath; - public EnvFileKeyValuePairs(string filePath) + public EnvFileKeyValuePairs(IExecutionContext context, string filePath) { + _context = context; _filePath = filePath; } public IEnumerator> GetEnumerator() { - var text = File.ReadAllText(_filePath) ?? string.Empty; + var text = string.Empty; + try + { + text = File.ReadAllText(_filePath) ?? string.Empty; + } + catch (DirectoryNotFoundException) + { + _context.Debug($"File does not exist '{_filePath}'"); + yield break; + } + catch (FileNotFoundException) + { + _context.Debug($"File does not exist '{_filePath}'"); + yield break; + } + var index = 0; var line = ReadLine(text, ref index); while (line != null) @@ -427,7 +369,7 @@ public IEnumerator> GetEnumerator() System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { - // Invoke IEnumerator GetEnumerator() above. + // Invoke IEnumerator> GetEnumerator() above. return GetEnumerator(); } @@ -470,7 +412,6 @@ private static string ReadLine( index = lfIndex + 1; // Skip over LF newline = "\n"; - var reader = new EnvFileKeyValuePairs("path"); return text.Substring(originalIndex, lfIndex - originalIndex); } }