From 31fe65b80df6d6ae731eff2795d5841691252f6d Mon Sep 17 00:00:00 2001
From: Jon Sequeira <jonsequeira@gmail.com>
Date: Tue, 19 Apr 2022 10:39:07 -0700
Subject: [PATCH 1/5] remove ParseArgsAsSpaceSeparated, small refactor of
 tokenizer

---
 .../ResponseFileTests.cs                      | 41 +++++----
 .../TokenizerInterpolationTests.cs            | 13 +++
 .../Parsing/ResponseFileHandling.cs           |  5 --
 .../Parsing/StringExtensions.cs               | 90 ++++++++-----------
 4 files changed, 71 insertions(+), 78 deletions(-)
 create mode 100644 src/System.CommandLine.Tests/TokenizerInterpolationTests.cs

diff --git a/src/System.CommandLine.Tests/ResponseFileTests.cs b/src/System.CommandLine.Tests/ResponseFileTests.cs
index 2bf64ad29f..cbaa7fd871 100644
--- a/src/System.CommandLine.Tests/ResponseFileTests.cs
+++ b/src/System.CommandLine.Tests/ResponseFileTests.cs
@@ -24,7 +24,7 @@ public void Dispose()
             }
         }
 
-        private string ResponseFile(params string[] lines)
+        private string CreateResponseFile(params string[] lines)
         {
             var responseFile = new FileInfo(Path.GetTempFileName());
 
@@ -46,7 +46,7 @@ public void When_response_file_specified_it_loads_options_from_response_file()
         {
             var option = new Option<bool>("--flag");
 
-            var result = option.Parse($"@{ResponseFile("--flag")}");
+            var result = option.Parse($"@{CreateResponseFile("--flag")}");
 
             result.HasOption(option).Should().BeTrue();
         }
@@ -54,7 +54,7 @@ public void When_response_file_specified_it_loads_options_from_response_file()
         [Fact]
         public void When_response_file_is_specified_it_loads_options_with_arguments_from_response_file()
         {
-            var responseFile = ResponseFile(
+            var responseFile = CreateResponseFile(
                 "--flag",
                 "--flag2",
                 "123");
@@ -77,7 +77,7 @@ public void When_response_file_is_specified_it_loads_options_with_arguments_from
         [Fact]
         public void When_response_file_is_specified_it_loads_command_arguments_from_response_file()
         {
-            var responseFile = ResponseFile(
+            var responseFile = CreateResponseFile(
                 "one",
                 "two",
                 "three");
@@ -98,7 +98,7 @@ public void When_response_file_is_specified_it_loads_command_arguments_from_resp
         [Fact]
         public void Response_file_can_provide_subcommand_arguments()
         {
-            var responseFile = ResponseFile(
+            var responseFile = CreateResponseFile(
                 "one",
                 "two",
                 "three");
@@ -122,7 +122,7 @@ public void Response_file_can_provide_subcommand_arguments()
         [Fact]
         public void Response_file_can_provide_subcommand()
         {
-            var responseFile = ResponseFile("subcommand");
+            var responseFile = CreateResponseFile("subcommand");
 
             var result = new RootCommand
                          {
@@ -143,7 +143,7 @@ public void Response_file_can_provide_subcommand()
         [Fact]
         public void When_response_file_is_specified_it_loads_subcommand_arguments_from_response_file()
         {
-            var responseFile = ResponseFile(
+            var responseFile = CreateResponseFile(
                 "one",
                 "two",
                 "three");
@@ -167,7 +167,7 @@ public void When_response_file_is_specified_it_loads_subcommand_arguments_from_r
         [Fact]
         public void Response_file_can_contain_blank_lines()
         {
-            var responseFile = ResponseFile(
+            var responseFile = CreateResponseFile(
                 "--flag",
                 "",
                 "123");
@@ -190,7 +190,7 @@ public void Response_file_can_contain_comments_which_are_ignored_when_loaded()
             var optionOne = new Option<bool>("--flag");
             var optionTwo = new Option<bool>("--flag2");
 
-            var responseFile = ResponseFile(
+            var responseFile = CreateResponseFile(
                 "# comment one",
                 "--flag",
                 "# comment two",
@@ -278,7 +278,7 @@ public void When_response_file_cannot_be_read_then_specified_error_is_returned()
         [InlineData("--flag=\"first value\" --flag2=123")]
         public void When_response_file_parse_as_space_separated_returns_expected_values(string input)
         {
-            var responseFile = ResponseFile(input);
+            var responseFile = CreateResponseFile(input);
 
             var optionOne = new Option<string>("--flag");
             var optionTwo = new Option<int>("--flag2");
@@ -289,7 +289,6 @@ public void When_response_file_parse_as_space_separated_returns_expected_values(
                 optionTwo
             };
             var parser = new CommandLineBuilder(rootCommand)
-                         .ParseResponseFileAs(ResponseFileHandling.ParseArgsAsSpaceSeparated)
                          .Build();
 
             var result = parser.Parse($"@{responseFile}");
@@ -322,9 +321,9 @@ public void When_response_file_processing_is_disabled_then_it_returns_response_f
         [Fact]
         public void Response_files_can_refer_to_other_response_files()
         {
-            var file3 = ResponseFile("--three", "3");
-            var file2 = ResponseFile($"@{file3}", "--two", "2");
-            var file1 = ResponseFile("--one", "1", $"@{file2}");
+            var file3 = CreateResponseFile("--three", "3");
+            var file2 = CreateResponseFile($"@{file3}", "--two", "2");
+            var file1 = CreateResponseFile("--one", "1", $"@{file2}");
 
             var option1 = new Option<int>("--one");
             var option2 = new Option<int>("--two");
@@ -339,17 +338,17 @@ public void Response_files_can_refer_to_other_response_files()
 
             var result = command.Parse($"@{file1}");
 
-            result.FindResultFor(option1).GetValueOrDefault().Should().Be(1);
-            result.FindResultFor(option1).GetValueOrDefault().Should().Be(1);
-            result.FindResultFor(option2).GetValueOrDefault().Should().Be(2);
-            result.FindResultFor(option3).GetValueOrDefault().Should().Be(3);
+            result.GetValueForOption(option1).Should().Be(1);
+            result.GetValueForOption(option1).Should().Be(1);
+            result.GetValueForOption(option2).Should().Be(2);
+            result.GetValueForOption(option3).Should().Be(3);
             result.Errors.Should().BeEmpty();
         }
 
         [Fact]
         public void When_response_file_options_or_arguments_contain_trailing_spaces_they_are_ignored()
         {
-            var responseFile = ResponseFile("--option1 ", "value1 ", "--option2\t", "2\t");
+            var responseFile = CreateResponseFile("--option1 ", "value1 ", "--option2\t", "2\t");
 
             var option1 = new Option<string>("--option1");
             var option2 = new Option<int>("--option2");
@@ -362,7 +361,7 @@ public void When_response_file_options_or_arguments_contain_trailing_spaces_they
         [Fact]
         public void When_response_file_options_or_arguments_contain_leading_spaces_they_are_ignored()
         {
-            var responseFile = ResponseFile(" --option1", " value1", "\t--option2", "\t2");
+            var responseFile = CreateResponseFile(" --option1", " value1", "\t--option2", "\t2");
 
             var option1 = new Option<string>("--option1");
             var option2 = new Option<int>("--option2");
@@ -376,7 +375,7 @@ public void When_response_file_options_or_arguments_contain_leading_spaces_they_
         [Fact]
         public void When_response_file_options_or_arguments_contain_trailing_and_leading_spaces_they_are_ignored()
         {
-            var responseFile = ResponseFile(" --option1 ", " value1 ", "\t--option2\t", "\t2\t");
+            var responseFile = CreateResponseFile(" --option1 ", " value1 ", "\t--option2\t", "\t2\t");
 
             var option1 = new Option<string>("--option1");
             var option2 = new Option<int>("--option2");
diff --git a/src/System.CommandLine.Tests/TokenizerInterpolationTests.cs b/src/System.CommandLine.Tests/TokenizerInterpolationTests.cs
new file mode 100644
index 0000000000..ec1432d863
--- /dev/null
+++ b/src/System.CommandLine.Tests/TokenizerInterpolationTests.cs
@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace System.CommandLine.Tests;
+
+public class TokenizerInterpolationTests
+{
+
+
+
+
+
+}
\ No newline at end of file
diff --git a/src/System.CommandLine/Parsing/ResponseFileHandling.cs b/src/System.CommandLine/Parsing/ResponseFileHandling.cs
index daa64d1c28..1caad45237 100644
--- a/src/System.CommandLine/Parsing/ResponseFileHandling.cs
+++ b/src/System.CommandLine/Parsing/ResponseFileHandling.cs
@@ -17,11 +17,6 @@ public enum ResponseFileHandling
         ///</remarks>
         ParseArgsAsLineSeparated,
 
-        ///<summary>
-        /// Arguments are separated by whitespace (spaces and/or new-lines)
-        ///</summary>
-        ParseArgsAsSpaceSeparated,
-
         ///<summary>
         /// Do not parse response files. Command line tokens beginning with <c>@</c> receive no special treatment.
         ///</summary>
diff --git a/src/System.CommandLine/Parsing/StringExtensions.cs b/src/System.CommandLine/Parsing/StringExtensions.cs
index 370c17c2fa..43632451ae 100644
--- a/src/System.CommandLine/Parsing/StringExtensions.cs
+++ b/src/System.CommandLine/Parsing/StringExtensions.cs
@@ -4,7 +4,6 @@
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
-using System.Linq;
 
 namespace System.CommandLine.Parsing
 {
@@ -129,7 +128,7 @@ internal static TokenizeResult Tokenize(
                 if (configuration.ResponseFileHandling != ResponseFileHandling.Disabled &&
                     arg.GetResponseFileReference() is { } filePath)
                 {
-                    ReadResponseFile(filePath, i);
+                    ReadResponseFile(filePath, i, configuration, argList, errorList);
                     continue;
                 }
 
@@ -279,37 +278,7 @@ bool PreviousTokenIsAnOptionExpectingAnArgument(out Option? option)
                 return false;
             }
 
-            void ReadResponseFile(string filePath, int i)
-            {
-                try
-                {
-                    var next = i + 1;
-
-                    foreach (var newArg in ExpandResponseFile(
-                        filePath,
-                        configuration.ResponseFileHandling))
-                    {
-                        argList.Insert(next, newArg);
-                        next += 1;
-                    }
-                }
-                catch (FileNotFoundException)
-                {
-                    var message = configuration.LocalizationResources
-                                               .ResponseFileNotFound(filePath);
-
-                    errorList.Add(
-                        new TokenizeError(message));
-                }
-                catch (IOException e)
-                {
-                    var message = configuration.LocalizationResources
-                                               .ErrorReadingResponseFile(filePath, e);
-
-                    errorList.Add(
-                        new TokenizeError(message));
-                }
-            }
+         
         }
 
         private static List<string> NormalizeRootCommand(
@@ -390,9 +359,40 @@ internal static bool TrySplitIntoSubtokens(
             return false;
         }
 
-        private static IEnumerable<string> ExpandResponseFile(
+        static void ReadResponseFile(
             string filePath,
-            ResponseFileHandling responseFileHandling)
+            int startAtIndex, 
+            CommandLineConfiguration configuration, 
+            List<string> argList, 
+            List<TokenizeError> errorList)
+        {
+            try
+            {
+                var next = startAtIndex + 1;
+
+                foreach (var newArg in ExpandResponseFile(filePath))
+                {
+                    argList.Insert(next, newArg);
+                    next += 1;
+                }
+            }
+            catch (FileNotFoundException)
+            {
+                var message = configuration.LocalizationResources
+                                           .ResponseFileNotFound(filePath);
+
+                errorList.Add(new TokenizeError(message));
+            }
+            catch (IOException e)
+            {
+                var message = configuration.LocalizationResources
+                                           .ErrorReadingResponseFile(filePath, e);
+
+                errorList.Add(new TokenizeError(message));
+            }
+        }
+
+        private static IEnumerable<string> ExpandResponseFile(string filePath)
         {
             var lines = File.ReadAllLines(filePath);
 
@@ -404,9 +404,7 @@ private static IEnumerable<string> ExpandResponseFile(
                 {
                     if (p.GetResponseFileReference() is { } path)
                     {
-                        foreach (var q in ExpandResponseFile(
-                            path,
-                            responseFileHandling))
+                        foreach (var q in ExpandResponseFile(path))
                         {
                             yield return q;
                         }
@@ -427,21 +425,9 @@ IEnumerable<string> SplitLine(string line)
                     yield break;
                 }
 
-                switch (responseFileHandling)
+                foreach (var word in CommandLineStringSplitter.Instance.Split(arg))
                 {
-                    case ResponseFileHandling.ParseArgsAsLineSeparated:
-
-                        yield return arg;
-
-                        break;
-                    case ResponseFileHandling.ParseArgsAsSpaceSeparated:
-
-                        foreach (var word in CommandLineStringSplitter.Instance.Split(arg))
-                        {
-                            yield return word;
-                        }
-
-                        break;
+                    yield return word;
                 }
             }
         }

From 920d9ebb2fc9383f2ea4e72355af89a7d6d3fcf0 Mon Sep 17 00:00:00 2001
From: Jon Sequeira <jonsequeira@gmail.com>
Date: Tue, 19 Apr 2022 18:44:19 -0700
Subject: [PATCH 2/5] generalize token replacement, implement response files as
 the default

---
 .../CustomTokenReplacerTests.cs               | 36 +++++++
 .../ResponseFileTests.cs                      |  3 +-
 .../TokenizerInterpolationTests.cs            | 13 ---
 .../Builder/CommandLineBuilder.cs             | 34 +++----
 .../Builder/CommandLineBuilderExtensions.cs   | 23 ++---
 .../CommandLineConfiguration.cs               | 34 +++++--
 .../Parsing/ParseArgument{T}.cs               | 21 ++--
 .../Parsing/ParseResultVisitor.cs             |  2 +-
 .../Parsing/ResponseFileHandling.cs           | 25 -----
 .../Parsing/StringExtensions.cs               | 97 ++++++++++---------
 .../Parsing/TokenizeError.cs                  | 24 -----
 .../Parsing/TokenizeResult.cs                 | 25 +++--
 .../Parsing/TryReplaceToken.cs                | 11 +++
 src/System.CommandLine/SymbolExtensions.cs    |  1 -
 14 files changed, 176 insertions(+), 173 deletions(-)
 create mode 100644 src/System.CommandLine.Tests/CustomTokenReplacerTests.cs
 delete mode 100644 src/System.CommandLine.Tests/TokenizerInterpolationTests.cs
 delete mode 100644 src/System.CommandLine/Parsing/ResponseFileHandling.cs
 delete mode 100644 src/System.CommandLine/Parsing/TokenizeError.cs
 create mode 100644 src/System.CommandLine/Parsing/TryReplaceToken.cs

diff --git a/src/System.CommandLine.Tests/CustomTokenReplacerTests.cs b/src/System.CommandLine.Tests/CustomTokenReplacerTests.cs
new file mode 100644
index 0000000000..f36219e692
--- /dev/null
+++ b/src/System.CommandLine.Tests/CustomTokenReplacerTests.cs
@@ -0,0 +1,36 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Collections.Generic;
+using System.CommandLine.Builder;
+using System.CommandLine.Parsing;
+using FluentAssertions;
+using Xunit;
+
+namespace System.CommandLine.Tests;
+
+public class CustomTokenReplacerTests
+{
+    [Fact]
+    public void Custom_token_replacer_can_expand_argument_values()
+    {
+        var argument = new Argument<int>();
+
+        var command = new RootCommand { argument };
+
+        var parser = new CommandLineBuilder(command)
+                     .UseTokenReplacer((string tokenToReplace, out IReadOnlyList<string> tokens, out string message) =>
+                     {
+                         tokens = new[] { "123" };
+                         message = null;
+                         return true;
+                     })
+                     .Build();
+
+        var result = parser.Parse("@interpolate-me");
+
+        result.Errors.Should().BeEmpty();
+
+        result.GetValueForArgument(argument).Should().Be(123);
+    }
+}
\ No newline at end of file
diff --git a/src/System.CommandLine.Tests/ResponseFileTests.cs b/src/System.CommandLine.Tests/ResponseFileTests.cs
index cbaa7fd871..f50ac20f11 100644
--- a/src/System.CommandLine.Tests/ResponseFileTests.cs
+++ b/src/System.CommandLine.Tests/ResponseFileTests.cs
@@ -306,7 +306,8 @@ public void When_response_file_processing_is_disabled_then_it_returns_response_f
             };
             var configuration = new CommandLineConfiguration(
                 command,
-                responseFileHandling: ResponseFileHandling.Disabled);
+                enableTokenReplacement: false);
+            
             var parser = new Parser(configuration);
 
             var result = parser.Parse("@file.rsp");
diff --git a/src/System.CommandLine.Tests/TokenizerInterpolationTests.cs b/src/System.CommandLine.Tests/TokenizerInterpolationTests.cs
deleted file mode 100644
index ec1432d863..0000000000
--- a/src/System.CommandLine.Tests/TokenizerInterpolationTests.cs
+++ /dev/null
@@ -1,13 +0,0 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-namespace System.CommandLine.Tests;
-
-public class TokenizerInterpolationTests
-{
-
-
-
-
-
-}
\ No newline at end of file
diff --git a/src/System.CommandLine/Builder/CommandLineBuilder.cs b/src/System.CommandLine/Builder/CommandLineBuilder.cs
index 959fcecdca..92f6a42446 100644
--- a/src/System.CommandLine/Builder/CommandLineBuilder.cs
+++ b/src/System.CommandLine/Builder/CommandLineBuilder.cs
@@ -37,24 +37,21 @@ public CommandLineBuilder(Command? rootCommand = null)
         /// Determines whether the parser recognizes command line directives.
         /// </summary>
         /// <seealso cref="DirectiveCollection"/>
-        public bool EnableDirectives { get; set; } = true;
+        internal bool EnableDirectives { get; set; } = true;
 
         /// <summary>
         /// Determines whether the parser recognize and expands POSIX-style bundled options.
         /// </summary>
-        public bool EnablePosixBundling { get; set; } = true;
+        internal bool EnablePosixBundling { get; set; } = true;
+        
+        internal bool EnableTokenReplacement { get; set; } = true;
 
         /// <summary>
         /// Determines the behavior when parsing a double dash (<c>--</c>) in a command line.
         /// </summary>
         /// <remarks>When set to <see langword="true"/>, all tokens following <c>--</c> will be placed into the <see cref="ParseResult.UnparsedTokens"/> collection. When set to <see langword="false"/>, all tokens following <c>--</c> will be treated as command arguments, even if they match an existing option.</remarks>
-        public bool EnableLegacyDoubleDashBehavior { get; set; }
-
-        /// <summary>
-        /// Configures the parser's handling of response files. When enabled, a command line token beginning with <c>@</c> that is a valid file path will be expanded as though inserted into the command line. 
-        /// </summary>
-        public ResponseFileHandling ResponseFileHandling { get; set; }
-
+        internal bool EnableLegacyDoubleDashBehavior { get; set; }
+        
         internal void CustomizeHelpLayout(Action<HelpContext> customize) => 
             _customizeHelpBuilder = customize;
 
@@ -89,24 +86,25 @@ internal LocalizationResources LocalizationResources
             set => _localizationResources = value;
         }
 
+        internal TryReplaceToken TokenReplacer { get; set; }
+
         /// <summary>
         /// Creates a parser based on the configuration of the command line builder.
         /// </summary>
-        public Parser Build()
-        {
-            var parser = new Parser(
+        public Parser Build() =>
+            new(
                 new CommandLineConfiguration(
                     Command,
                     enablePosixBundling: EnablePosixBundling,
                     enableDirectives: EnableDirectives,
                     enableLegacyDoubleDashBehavior: EnableLegacyDoubleDashBehavior,
+                    enableTokenReplacement: EnableTokenReplacement,
                     resources: LocalizationResources,
-                    responseFileHandling: ResponseFileHandling,
-                    middlewarePipeline: _middlewareList is null ? Array.Empty<InvocationMiddleware>() : GetMiddleware(),
-                    helpBuilderFactory: GetHelpBuilderFactory()));
-            
-            return parser;
-        }
+                    middlewarePipeline: _middlewareList is null
+                                            ? Array.Empty<InvocationMiddleware>()
+                                            : GetMiddleware(),
+                    helpBuilderFactory: GetHelpBuilderFactory(),
+                    tokenReplacer: TokenReplacer));
 
         private IReadOnlyList<InvocationMiddleware> GetMiddleware()
         {
diff --git a/src/System.CommandLine/Builder/CommandLineBuilderExtensions.cs b/src/System.CommandLine/Builder/CommandLineBuilderExtensions.cs
index 1b2183da91..00a6e004bb 100644
--- a/src/System.CommandLine/Builder/CommandLineBuilderExtensions.cs
+++ b/src/System.CommandLine/Builder/CommandLineBuilderExtensions.cs
@@ -152,19 +152,7 @@ public static CommandLineBuilder EnablePosixBundling(
             builder.EnablePosixBundling = value;
             return builder;
         }
-
-        /// <inheritdoc cref="CommandLineBuilder.ResponseFileHandling"/>
-        /// <param name="responseFileHandling">Specifies whether or how response files are parsed.</param>
-        /// <param name="builder">A command line builder.</param>
-        /// <returns>The same instance of <see cref="CommandLineBuilder"/>.</returns>
-        public static CommandLineBuilder ParseResponseFileAs(
-            this CommandLineBuilder builder,
-            ResponseFileHandling responseFileHandling)
-        {
-            builder.ResponseFileHandling = responseFileHandling;
-            return builder;
-        }
-
+        
         /// <summary>
         /// Ensures that the application is registered with the <c>dotnet-suggest</c> tool to enable command line completions.
         /// </summary>
@@ -589,6 +577,15 @@ public static CommandLineBuilder UseLocalizationResources(
             return builder;
         }
 
+        public static CommandLineBuilder UseTokenReplacer(
+            this CommandLineBuilder builder,
+            TryReplaceToken? replaceToken)
+        {
+            builder.TokenReplacer = replaceToken;
+
+            return builder;
+        }
+
         /// <summary>
         /// Enables the use of a option (defaulting to the alias <c>--version</c>) which when specified in command line input will short circuit normal command handling and instead write out version information before exiting.
         /// </summary>
diff --git a/src/System.CommandLine/CommandLineConfiguration.cs b/src/System.CommandLine/CommandLineConfiguration.cs
index 301a15d515..1e2699298d 100644
--- a/src/System.CommandLine/CommandLineConfiguration.cs
+++ b/src/System.CommandLine/CommandLineConfiguration.cs
@@ -17,6 +17,7 @@ namespace System.CommandLine
     public class CommandLineConfiguration
     {
         private Func<BindingContext, HelpBuilder>? _helpBuilderFactory;
+        private TryReplaceToken? _tokenReplacer;
 
         /// <summary>
         /// Initializes a new instance of the CommandLineConfiguration class.
@@ -26,7 +27,6 @@ public class CommandLineConfiguration
         /// <param name="enableDirectives"><see langword="true"/> to enable directive parsing; otherwise, <see langword="false"/>.</param>
         /// <param name="enableLegacyDoubleDashBehavior">Enables the legacy behavior of the <c>--</c> token, which is to ignore parsing of subsequent tokens and place them in the <see cref="ParseResult.UnparsedTokens"/> list.</param>
         /// <param name="resources">Provide custom validation messages.</param>
-        /// <param name="responseFileHandling">One of the enumeration values that specifies how response files (.rsp) are handled.</param>
         /// <param name="middlewarePipeline">Provide a custom middleware pipeline.</param>
         /// <param name="helpBuilderFactory">Provide a custom help builder.</param>
         public CommandLineConfiguration(
@@ -34,21 +34,24 @@ public CommandLineConfiguration(
             bool enablePosixBundling = true,
             bool enableDirectives = true,
             bool enableLegacyDoubleDashBehavior = false,
+            bool enableTokenReplacement = true,
             LocalizationResources? resources = null,
-            ResponseFileHandling responseFileHandling = ResponseFileHandling.ParseArgsAsLineSeparated,
             IReadOnlyList<InvocationMiddleware>? middlewarePipeline = null,
-            Func<BindingContext, HelpBuilder>? helpBuilderFactory = null)
+            Func<BindingContext, HelpBuilder>? helpBuilderFactory = null,
+            TryReplaceToken? tokenReplacer = null)
         {
             RootCommand = command ?? throw new ArgumentNullException(nameof(command));
 
             EnableLegacyDoubleDashBehavior = enableLegacyDoubleDashBehavior;
+            EnableTokenReplacement = enableTokenReplacement;
             EnablePosixBundling = enablePosixBundling;
             EnableDirectives = enableDirectives;
+
             LocalizationResources = resources ?? LocalizationResources.Instance;
-            ResponseFileHandling = responseFileHandling;
             Middleware = middlewarePipeline ?? Array.Empty<InvocationMiddleware>();
 
             _helpBuilderFactory = helpBuilderFactory;
+            _tokenReplacer = tokenReplacer;
         }
 
         internal static HelpBuilder DefaultHelpBuilderFactory(BindingContext context, int? requestedMaxWidth = null)
@@ -73,29 +76,44 @@ internal static HelpBuilder DefaultHelpBuilderFactory(BindingContext context, in
         public bool EnableLegacyDoubleDashBehavior { get; }
 
         /// <summary>
-        /// Gets whether POSIX bundling is enabled.
+        /// Gets a value indicating whether POSIX bundling is enabled.
         /// </summary>
         /// <remarks>
         /// POSIX recommends that single-character options be allowed to be specified together after a single <c>-</c> prefix.
         /// </remarks>
         public bool EnablePosixBundling { get; }
+        
+        public bool EnableTokenReplacement { get; }
 
         /// <summary>
         /// Gets the localizable resources.
         /// </summary>
         public LocalizationResources LocalizationResources { get; }
 
-        internal Func<BindingContext, HelpBuilder> HelpBuilderFactory => _helpBuilderFactory ??= (context) => DefaultHelpBuilderFactory(context);
+        internal Func<BindingContext, HelpBuilder> HelpBuilderFactory => _helpBuilderFactory ??= context => DefaultHelpBuilderFactory(context);
 
         internal IReadOnlyList<InvocationMiddleware> Middleware { get; }
 
+        public TryReplaceToken? TokenReplacer =>
+            EnableTokenReplacement
+                ? _tokenReplacer ??= DefaultTokenReplacer
+                : null;
+
+        private bool DefaultTokenReplacer(
+            string tokenToReplace, 
+            out IReadOnlyList<string>? replacementTokens, 
+            out string? errorMessage) =>
+            StringExtensions.TryReadResponseFile(
+                tokenToReplace,
+                LocalizationResources,
+                out replacementTokens,
+                out errorMessage);
+
         /// <summary>
         /// Gets the root command.
         /// </summary>
         public Command RootCommand { get; }
 
-        internal ResponseFileHandling ResponseFileHandling { get; }
-
         /// <summary>
         /// Throws an exception if the parser configuration is ambiguous or otherwise not valid.
         /// </summary>
diff --git a/src/System.CommandLine/Parsing/ParseArgument{T}.cs b/src/System.CommandLine/Parsing/ParseArgument{T}.cs
index 6c434e0212..5a4f59c2b4 100644
--- a/src/System.CommandLine/Parsing/ParseArgument{T}.cs
+++ b/src/System.CommandLine/Parsing/ParseArgument{T}.cs
@@ -1,14 +1,13 @@
 // Copyright (c) .NET Foundation and contributors. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-namespace System.CommandLine.Parsing
-{
-    /// <summary>
-    /// Performs custom parsing of an argument.
-    /// </summary>
-    /// <typeparam name="T">The type which the argument is to be parsed as.</typeparam>
-    /// <param name="result">The argument result.</param>
-    /// <returns>The parsed value.</returns>
-    /// <remarks>Validation errors can be returned by setting <see cref="SymbolResult.ErrorMessage"/>.</remarks>
-    public delegate T ParseArgument<out T>(ArgumentResult result);
-}
\ No newline at end of file
+namespace System.CommandLine.Parsing;
+
+/// <summary>
+/// Performs custom parsing of an argument.
+/// </summary>
+/// <typeparam name="T">The type which the argument is to be parsed as.</typeparam>
+/// <param name="result">The argument result.</param>
+/// <returns>The parsed value.</returns>
+/// <remarks>Validation errors can be returned by setting <see cref="SymbolResult.ErrorMessage"/>.</remarks>
+public delegate T ParseArgument<out T>(ArgumentResult result);
\ No newline at end of file
diff --git a/src/System.CommandLine/Parsing/ParseResultVisitor.cs b/src/System.CommandLine/Parsing/ParseResultVisitor.cs
index 3cb4a7f955..b3fe04cbd3 100644
--- a/src/System.CommandLine/Parsing/ParseResultVisitor.cs
+++ b/src/System.CommandLine/Parsing/ParseResultVisitor.cs
@@ -45,7 +45,7 @@ public ParseResultVisitor(
             for (var i = 0; i < _tokenizeResult.Errors.Count; i++)
             {
                 var error = _tokenizeResult.Errors[i];
-                _errors.Add(new ParseError(error.Message));
+                _errors.Add(new ParseError(error));
             }
         }
 
diff --git a/src/System.CommandLine/Parsing/ResponseFileHandling.cs b/src/System.CommandLine/Parsing/ResponseFileHandling.cs
deleted file mode 100644
index 1caad45237..0000000000
--- a/src/System.CommandLine/Parsing/ResponseFileHandling.cs
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-namespace System.CommandLine.Parsing
-{
-    /// <summary>
-    /// Specifies settings for response file parsing.
-    /// </summary>
-    public enum ResponseFileHandling
-    {
-
-        ///<summary>
-        /// Each line in the file is treated as a single argument, regardless of whitespace on the line.
-        ///</summary>
-        ///<remarks>
-        /// Empty lines and lines beginning with <c>#</c> are skipped.
-        ///</remarks>
-        ParseArgsAsLineSeparated,
-
-        ///<summary>
-        /// Do not parse response files. Command line tokens beginning with <c>@</c> receive no special treatment.
-        ///</summary>
-        Disabled
-    }
-}
diff --git a/src/System.CommandLine/Parsing/StringExtensions.cs b/src/System.CommandLine/Parsing/StringExtensions.cs
index 43632451ae..bf15c52aa9 100644
--- a/src/System.CommandLine/Parsing/StringExtensions.cs
+++ b/src/System.CommandLine/Parsing/StringExtensions.cs
@@ -4,6 +4,7 @@
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
+using System.Linq;
 
 namespace System.CommandLine.Parsing
 {
@@ -35,8 +36,11 @@ internal static int GetPrefixLength(this string alias)
         {
             if (alias[0] == '-')
             {
-                return alias.Length > 1 && alias[1] == '-' ? 2 : 1;
+                return alias.Length > 1 && alias[1] == '-' 
+                           ? 2 
+                           : 1;
             }
+
             if (alias[0] == '/')
             {
                 return 1;
@@ -69,14 +73,13 @@ internal static TokenizeResult Tokenize(
             CommandLineConfiguration configuration,
             bool inferRootCommand = true)
         {
-            var errorList = new List<TokenizeError>();
+            var errorList = new List<string>();
 
-            Command currentCommand = configuration.RootCommand;
+            var currentCommand = configuration.RootCommand;
             var foundDoubleDash = false;
             var foundEndOfDirectives = !configuration.EnableDirectives;
-            
-            List<string> argList;
-            argList = NormalizeRootCommand(args, configuration.RootCommand, inferRootCommand);
+
+            var argList = NormalizeRootCommand(args, configuration.RootCommand, inferRootCommand);
             
             var tokenList = new List<Token>(argList.Count);
 
@@ -125,10 +128,22 @@ internal static TokenizeResult Tokenize(
                     }
                 }
 
-                if (configuration.ResponseFileHandling != ResponseFileHandling.Disabled &&
-                    arg.GetResponseFileReference() is { } filePath)
+                if (configuration.EnableTokenReplacement &&
+                    configuration.TokenReplacer is { } replacer &&
+                    arg.GetReplaceableTokenValue() is { } value)
                 {
-                    ReadResponseFile(filePath, i, configuration, argList, errorList);
+                    if (replacer(
+                            value,
+                            out var newTokens,
+                            out var error))
+                    {
+                        argList.InsertRange(i + 1, newTokens!);
+                    }
+                    else
+                    {
+                        errorList.Add(error!);
+                    }
+
                     continue;
                 }
 
@@ -330,7 +345,7 @@ private static List<string> NormalizeRootCommand(
             return list;
         }
 
-        private static string? GetResponseFileReference(this string arg) =>
+        private static string? GetReplaceableTokenValue(this string arg) =>
             arg.Length > 1 && arg[0] == '@'
                 ? arg.Substring(1)
                 : null;
@@ -359,64 +374,56 @@ internal static bool TrySplitIntoSubtokens(
             return false;
         }
 
-        static void ReadResponseFile(
+        internal static bool TryReadResponseFile(
             string filePath,
-            int startAtIndex, 
-            CommandLineConfiguration configuration, 
-            List<string> argList, 
-            List<TokenizeError> errorList)
+            LocalizationResources localizationResources,
+            out IReadOnlyList<string>? newTokens,
+            out string? error)
         {
             try
             {
-                var next = startAtIndex + 1;
-
-                foreach (var newArg in ExpandResponseFile(filePath))
-                {
-                    argList.Insert(next, newArg);
-                    next += 1;
-                }
+                newTokens = ExpandResponseFile(filePath).ToArray();
+                error = null;
+                return true;
             }
             catch (FileNotFoundException)
             {
-                var message = configuration.LocalizationResources
-                                           .ResponseFileNotFound(filePath);
-
-                errorList.Add(new TokenizeError(message));
+                error = localizationResources.ResponseFileNotFound(filePath);
             }
             catch (IOException e)
             {
-                var message = configuration.LocalizationResources
-                                           .ErrorReadingResponseFile(filePath, e);
-
-                errorList.Add(new TokenizeError(message));
+                error = localizationResources.ErrorReadingResponseFile(filePath, e);
             }
-        }
 
-        private static IEnumerable<string> ExpandResponseFile(string filePath)
-        {
-            var lines = File.ReadAllLines(filePath);
+            newTokens = null;
+            return false;
 
-            for (var i = 0; i < lines.Length; i++)
+            static IEnumerable<string> ExpandResponseFile(string filePath)
             {
-                var line = lines[i];
+                var lines = File.ReadAllLines(filePath);
 
-                foreach (var p in SplitLine(line))
+                for (var i = 0; i < lines.Length; i++)
                 {
-                    if (p.GetResponseFileReference() is { } path)
+                    var line = lines[i];
+
+                    foreach (var p in SplitLine(line))
                     {
-                        foreach (var q in ExpandResponseFile(path))
+                        if (p.GetReplaceableTokenValue() is { } path)
                         {
-                            yield return q;
+                            foreach (var q in ExpandResponseFile(path))
+                            {
+                                yield return q;
+                            }
+                        }
+                        else
+                        {
+                            yield return p;
                         }
-                    }
-                    else
-                    {
-                        yield return p;
                     }
                 }
             }
 
-            IEnumerable<string> SplitLine(string line)
+            static IEnumerable<string> SplitLine(string line)
             {
                 var arg = line.Trim();
 
diff --git a/src/System.CommandLine/Parsing/TokenizeError.cs b/src/System.CommandLine/Parsing/TokenizeError.cs
deleted file mode 100644
index 6af650c20d..0000000000
--- a/src/System.CommandLine/Parsing/TokenizeError.cs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-namespace System.CommandLine.Parsing
-{
-    /// <summary>
-    /// Describes an error that occurs while tokenizing command line input.
-    /// </summary>
-    public class TokenizeError
-    {
-        internal TokenizeError(string message)
-        {
-            Message = message ?? throw new ArgumentNullException(nameof(message));
-        }
-
-        /// <summary>
-        /// A message to explain the error to a user.
-        /// </summary>
-        public string Message { get; }
-
-        /// <inheritdoc />
-        public override string ToString() => Message;
-    }
-}
diff --git a/src/System.CommandLine/Parsing/TokenizeResult.cs b/src/System.CommandLine/Parsing/TokenizeResult.cs
index 0fbd5dcfca..565a0cb8e7 100644
--- a/src/System.CommandLine/Parsing/TokenizeResult.cs
+++ b/src/System.CommandLine/Parsing/TokenizeResult.cs
@@ -3,20 +3,19 @@
 
 using System.Collections.Generic;
 
-namespace System.CommandLine.Parsing
+namespace System.CommandLine.Parsing;
+
+internal class TokenizeResult
 {
-    internal class TokenizeResult
+    internal TokenizeResult(
+        List<Token> tokens,
+        List<string> errors)
     {
-        internal TokenizeResult(
-            List<Token> tokens,
-            List<TokenizeError> errors)
-        {
-            Tokens = tokens;
-            Errors = errors;
-        }
+        Tokens = tokens;
+        Errors = errors;
+    }
 
-        public List<Token> Tokens { get; }
+    public List<Token> Tokens { get; }
 
-        public List<TokenizeError> Errors { get; }
-    }
-}
+    public List<string> Errors { get; }
+}
\ No newline at end of file
diff --git a/src/System.CommandLine/Parsing/TryReplaceToken.cs b/src/System.CommandLine/Parsing/TryReplaceToken.cs
new file mode 100644
index 0000000000..d515f52b17
--- /dev/null
+++ b/src/System.CommandLine/Parsing/TryReplaceToken.cs
@@ -0,0 +1,11 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Collections.Generic;
+
+namespace System.CommandLine.Parsing;
+
+public delegate bool TryReplaceToken(
+    string tokenToReplace,
+    out IReadOnlyList<string>? replacementTokens, 
+    out string? errorMessage);
\ No newline at end of file
diff --git a/src/System.CommandLine/SymbolExtensions.cs b/src/System.CommandLine/SymbolExtensions.cs
index 74da58dbb3..6739fb2e4c 100644
--- a/src/System.CommandLine/SymbolExtensions.cs
+++ b/src/System.CommandLine/SymbolExtensions.cs
@@ -4,7 +4,6 @@
 using System.Collections.Generic;
 using System.CommandLine.Builder;
 using System.CommandLine.Parsing;
-using System.Linq;
 
 namespace System.CommandLine
 {

From 5b6fed5c5e092b4b47b861ab8733097efc542a36 Mon Sep 17 00:00:00 2001
From: Jon Sequeira <jonsequeira@gmail.com>
Date: Wed, 20 Apr 2022 08:45:30 -0700
Subject: [PATCH 3/5] add XML doc comments

---
 src/System.CommandLine/Builder/CommandLineBuilder.cs |  2 +-
 .../Builder/CommandLineBuilderExtensions.cs          |  6 ++++++
 src/System.CommandLine/CommandLineConfiguration.cs   | 12 ++++++++++--
 src/System.CommandLine/Parsing/TryReplaceToken.cs    |  3 +++
 4 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/System.CommandLine/Builder/CommandLineBuilder.cs b/src/System.CommandLine/Builder/CommandLineBuilder.cs
index 92f6a42446..7fd52454af 100644
--- a/src/System.CommandLine/Builder/CommandLineBuilder.cs
+++ b/src/System.CommandLine/Builder/CommandLineBuilder.cs
@@ -86,7 +86,7 @@ internal LocalizationResources LocalizationResources
             set => _localizationResources = value;
         }
 
-        internal TryReplaceToken TokenReplacer { get; set; }
+        internal TryReplaceToken? TokenReplacer { get; set; }
 
         /// <summary>
         /// Creates a parser based on the configuration of the command line builder.
diff --git a/src/System.CommandLine/Builder/CommandLineBuilderExtensions.cs b/src/System.CommandLine/Builder/CommandLineBuilderExtensions.cs
index 00a6e004bb..132fe99943 100644
--- a/src/System.CommandLine/Builder/CommandLineBuilderExtensions.cs
+++ b/src/System.CommandLine/Builder/CommandLineBuilderExtensions.cs
@@ -577,6 +577,12 @@ public static CommandLineBuilder UseLocalizationResources(
             return builder;
         }
 
+        /// <summary>
+        /// Specifies a delegate used to replace any token prefixed with <code>@</code> with zero or more other tokens, prior to parsing. 
+        /// </summary>
+        /// <param name="builder">A command line builder.</param>
+        /// <param name="replaceToken">Replaces the specified token with any number of other tokens.</param>
+        /// <returns>The same instance of <see cref="CommandLineBuilder"/>.</returns>
         public static CommandLineBuilder UseTokenReplacer(
             this CommandLineBuilder builder,
             TryReplaceToken? replaceToken)
diff --git a/src/System.CommandLine/CommandLineConfiguration.cs b/src/System.CommandLine/CommandLineConfiguration.cs
index 1e2699298d..85e9eeb6f3 100644
--- a/src/System.CommandLine/CommandLineConfiguration.cs
+++ b/src/System.CommandLine/CommandLineConfiguration.cs
@@ -26,9 +26,11 @@ public class CommandLineConfiguration
         /// <param name="enablePosixBundling"><see langword="true"/> to enable POSIX bundling; otherwise, <see langword="false"/>.</param>
         /// <param name="enableDirectives"><see langword="true"/> to enable directive parsing; otherwise, <see langword="false"/>.</param>
         /// <param name="enableLegacyDoubleDashBehavior">Enables the legacy behavior of the <c>--</c> token, which is to ignore parsing of subsequent tokens and place them in the <see cref="ParseResult.UnparsedTokens"/> list.</param>
+        /// <param name="enableTokenReplacement"><see langword="true"/> to enable token replacement; otherwise, <see langword="false"/>.</param>
         /// <param name="resources">Provide custom validation messages.</param>
         /// <param name="middlewarePipeline">Provide a custom middleware pipeline.</param>
         /// <param name="helpBuilderFactory">Provide a custom help builder.</param>
+        /// <param name="tokenReplacer">Replaces the specified token with any number of other tokens.</param>
         public CommandLineConfiguration(
             Command command,
             bool enablePosixBundling = true,
@@ -82,7 +84,13 @@ internal static HelpBuilder DefaultHelpBuilderFactory(BindingContext context, in
         /// POSIX recommends that single-character options be allowed to be specified together after a single <c>-</c> prefix.
         /// </remarks>
         public bool EnablePosixBundling { get; }
-        
+
+        /// <summary>
+        /// Gets a value indicating whether token replacement is enabled.
+        /// </summary>
+        /// <remarks>
+        /// When enabled, any token prefixed with <code>@</code> can be replaced with zero or more other tokens. This is mostly commonly used to expand tokens from response files and interpolate them into a command line prior to parsing.
+        /// </remarks>
         public bool EnableTokenReplacement { get; }
 
         /// <summary>
@@ -94,7 +102,7 @@ internal static HelpBuilder DefaultHelpBuilderFactory(BindingContext context, in
 
         internal IReadOnlyList<InvocationMiddleware> Middleware { get; }
 
-        public TryReplaceToken? TokenReplacer =>
+        internal TryReplaceToken? TokenReplacer =>
             EnableTokenReplacement
                 ? _tokenReplacer ??= DefaultTokenReplacer
                 : null;
diff --git a/src/System.CommandLine/Parsing/TryReplaceToken.cs b/src/System.CommandLine/Parsing/TryReplaceToken.cs
index d515f52b17..2ddae16312 100644
--- a/src/System.CommandLine/Parsing/TryReplaceToken.cs
+++ b/src/System.CommandLine/Parsing/TryReplaceToken.cs
@@ -5,6 +5,9 @@
 
 namespace System.CommandLine.Parsing;
 
+/// <summary>
+/// Replaces a token with one or more other tokens prior to parsing.
+/// </summary>
 public delegate bool TryReplaceToken(
     string tokenToReplace,
     out IReadOnlyList<string>? replacementTokens, 

From 49324078e146fa2ac08930fb59a0f050f55e58c3 Mon Sep 17 00:00:00 2001
From: Jon Sequeira <jonsequeira@gmail.com>
Date: Wed, 20 Apr 2022 08:49:58 -0700
Subject: [PATCH 4/5] fix API compat tests

---
 ...ommandLine_api_is_not_changed.approved.txt | 21 +++++++------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/src/System.CommandLine.ApiCompatibility.Tests/ApiCompatibilityApprovalTests.System_CommandLine_api_is_not_changed.approved.txt b/src/System.CommandLine.ApiCompatibility.Tests/ApiCompatibilityApprovalTests.System_CommandLine_api_is_not_changed.approved.txt
index 32365a95ab..56a0a185ff 100644
--- a/src/System.CommandLine.ApiCompatibility.Tests/ApiCompatibilityApprovalTests.System_CommandLine_api_is_not_changed.approved.txt
+++ b/src/System.CommandLine.ApiCompatibility.Tests/ApiCompatibilityApprovalTests.System_CommandLine_api_is_not_changed.approved.txt
@@ -71,10 +71,11 @@
     public static System.CommandLine.Parsing.ParseResult Parse(this Command command, System.String[] args)
     public static System.CommandLine.Parsing.ParseResult Parse(this Command command, System.String commandLine)
   public class CommandLineConfiguration
-    .ctor(Command command, System.Boolean enablePosixBundling = True, System.Boolean enableDirectives = True, System.Boolean enableLegacyDoubleDashBehavior = False, LocalizationResources resources = null, System.CommandLine.Parsing.ResponseFileHandling responseFileHandling = ParseArgsAsLineSeparated, System.Collections.Generic.IReadOnlyList<System.CommandLine.Invocation.InvocationMiddleware> middlewarePipeline = null, System.Func<System.CommandLine.Binding.BindingContext,System.CommandLine.Help.HelpBuilder> helpBuilderFactory = null)
+    .ctor(Command command, System.Boolean enablePosixBundling = True, System.Boolean enableDirectives = True, System.Boolean enableLegacyDoubleDashBehavior = False, System.Boolean enableTokenReplacement = True, LocalizationResources resources = null, System.Collections.Generic.IReadOnlyList<System.CommandLine.Invocation.InvocationMiddleware> middlewarePipeline = null, System.Func<System.CommandLine.Binding.BindingContext,System.CommandLine.Help.HelpBuilder> helpBuilderFactory = null, System.CommandLine.Parsing.TryReplaceToken tokenReplacer = null)
     public System.Boolean EnableDirectives { get; }
     public System.Boolean EnableLegacyDoubleDashBehavior { get; }
     public System.Boolean EnablePosixBundling { get; }
+    public System.Boolean EnableTokenReplacement { get; }
     public LocalizationResources LocalizationResources { get; }
     public Command RootCommand { get; }
     public System.Void ThrowIfInvalid()
@@ -250,10 +251,6 @@ System.CommandLine.Builder
   public class CommandLineBuilder
     .ctor(System.CommandLine.Command rootCommand = null)
     public System.CommandLine.Command Command { get; }
-    public System.Boolean EnableDirectives { get; set; }
-    public System.Boolean EnableLegacyDoubleDashBehavior { get; set; }
-    public System.Boolean EnablePosixBundling { get; set; }
-    public System.CommandLine.Parsing.ResponseFileHandling ResponseFileHandling { get; set; }
     public System.CommandLine.Parsing.Parser Build()
   public static class CommandLineBuilderExtensions
     public static CommandLineBuilder AddMiddleware(this CommandLineBuilder builder, System.CommandLine.Invocation.InvocationMiddleware middleware, System.CommandLine.Invocation.MiddlewareOrder order = Default)
@@ -262,7 +259,6 @@ System.CommandLine.Builder
     public static CommandLineBuilder EnableDirectives(this CommandLineBuilder builder, System.Boolean value = True)
     public static CommandLineBuilder EnableLegacyDoubleDashBehavior(this CommandLineBuilder builder, System.Boolean value = True)
     public static CommandLineBuilder EnablePosixBundling(this CommandLineBuilder builder, System.Boolean value = True)
-    public static CommandLineBuilder ParseResponseFileAs(this CommandLineBuilder builder, System.CommandLine.Parsing.ResponseFileHandling responseFileHandling)
     public static CommandLineBuilder RegisterWithDotnetSuggest(this CommandLineBuilder builder)
     public static CommandLineBuilder UseDefaults(this CommandLineBuilder builder)
     public static CommandLineBuilder UseEnvironmentVariableDirective(this CommandLineBuilder builder)
@@ -275,6 +271,7 @@ System.CommandLine.Builder
     public static CommandLineBuilder UseParseDirective(this CommandLineBuilder builder, System.Nullable<System.Int32> errorExitCode = null)
     public static CommandLineBuilder UseParseErrorReporting(this CommandLineBuilder builder, System.Nullable<System.Int32> errorExitCode = null)
     public static CommandLineBuilder UseSuggestDirective(this CommandLineBuilder builder)
+    public static CommandLineBuilder UseTokenReplacer(this CommandLineBuilder builder, System.CommandLine.Parsing.TryReplaceToken replaceToken)
     public static CommandLineBuilder UseTypoCorrections(this CommandLineBuilder builder, System.Int32 maxLevenshteinDistance = 3)
     public static CommandLineBuilder UseVersionOption(this CommandLineBuilder builder)
     public static CommandLineBuilder UseVersionOption(this CommandLineBuilder builder, System.String[] aliases)
@@ -476,10 +473,6 @@ System.CommandLine.Parsing
     public static System.Threading.Tasks.Task<System.Int32> InvokeAsync(this Parser parser, System.String commandLine, System.CommandLine.IConsole console = null)
     public static System.Threading.Tasks.Task<System.Int32> InvokeAsync(this Parser parser, System.String[] args, System.CommandLine.IConsole console = null)
     public static ParseResult Parse(this Parser parser, System.String commandLine)
-  public enum ResponseFileHandling : System.Enum, System.IComparable, System.IConvertible, System.IFormattable
-    ParseArgsAsLineSeparated=0
-    ParseArgsAsSpaceSeparated=1
-    Disabled=2
   public abstract class SymbolResult
     public System.Collections.Generic.IReadOnlyList<SymbolResult> Children { get; }
     public System.String ErrorMessage { get; set; }
@@ -505,9 +498,6 @@ System.CommandLine.Parsing
     public System.Boolean Equals(Token other)
     public System.Int32 GetHashCode()
     public System.String ToString()
-  public class TokenizeError
-    public System.String Message { get; }
-    public System.String ToString()
   public enum TokenType : System.Enum, System.IComparable, System.IConvertible, System.IFormattable
     Argument=0
     Command=1
@@ -515,6 +505,11 @@ System.CommandLine.Parsing
     DoubleDash=3
     Unparsed=4
     Directive=5
+  public delegate TryReplaceToken : System.MulticastDelegate, System.ICloneable, System.Runtime.Serialization.ISerializable
+    .ctor(System.Object object, System.IntPtr method)
+    public System.IAsyncResult BeginInvoke(System.String tokenToReplace, ref System.Collections.Generic.IReadOnlyList<System.String> replacementTokens, ref System.String& errorMessage, System.AsyncCallback callback, System.Object object)
+    public System.Boolean EndInvoke(ref System.Collections.Generic.IReadOnlyList<System.String> replacementTokens, ref System.String& errorMessage, System.IAsyncResult result)
+    public System.Boolean Invoke(System.String tokenToReplace, ref System.Collections.Generic.IReadOnlyList<System.String> replacementTokens, ref System.String& errorMessage)
   public delegate ValidateSymbolResult<in T> : System.MulticastDelegate, System.ICloneable, System.Runtime.Serialization.ISerializable
     .ctor(System.Object object, System.IntPtr method)
     public System.IAsyncResult BeginInvoke(T symbolResult, System.AsyncCallback callback, System.Object object)

From 719e8b251c02ee37a6f70d7a23b2068dc9af5a87 Mon Sep 17 00:00:00 2001
From: Jon Sequeira <jonsequeira@gmail.com>
Date: Wed, 20 Apr 2022 14:24:35 -0700
Subject: [PATCH 5/5] add test cases and fix a couple of issues

---
 .../CustomTokenReplacerTests.cs               |  36 ---
 .../TokenReplacementTests.cs                  | 219 ++++++++++++++++++
 .../Parsing/StringExtensions.cs               |  10 +-
 3 files changed, 223 insertions(+), 42 deletions(-)
 delete mode 100644 src/System.CommandLine.Tests/CustomTokenReplacerTests.cs
 create mode 100644 src/System.CommandLine.Tests/TokenReplacementTests.cs

diff --git a/src/System.CommandLine.Tests/CustomTokenReplacerTests.cs b/src/System.CommandLine.Tests/CustomTokenReplacerTests.cs
deleted file mode 100644
index f36219e692..0000000000
--- a/src/System.CommandLine.Tests/CustomTokenReplacerTests.cs
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-using System.Collections.Generic;
-using System.CommandLine.Builder;
-using System.CommandLine.Parsing;
-using FluentAssertions;
-using Xunit;
-
-namespace System.CommandLine.Tests;
-
-public class CustomTokenReplacerTests
-{
-    [Fact]
-    public void Custom_token_replacer_can_expand_argument_values()
-    {
-        var argument = new Argument<int>();
-
-        var command = new RootCommand { argument };
-
-        var parser = new CommandLineBuilder(command)
-                     .UseTokenReplacer((string tokenToReplace, out IReadOnlyList<string> tokens, out string message) =>
-                     {
-                         tokens = new[] { "123" };
-                         message = null;
-                         return true;
-                     })
-                     .Build();
-
-        var result = parser.Parse("@interpolate-me");
-
-        result.Errors.Should().BeEmpty();
-
-        result.GetValueForArgument(argument).Should().Be(123);
-    }
-}
\ No newline at end of file
diff --git a/src/System.CommandLine.Tests/TokenReplacementTests.cs b/src/System.CommandLine.Tests/TokenReplacementTests.cs
new file mode 100644
index 0000000000..7bc72ea020
--- /dev/null
+++ b/src/System.CommandLine.Tests/TokenReplacementTests.cs
@@ -0,0 +1,219 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Collections.Generic;
+using System.CommandLine.Builder;
+using System.CommandLine.Parsing;
+using FluentAssertions;
+using Xunit;
+
+namespace System.CommandLine.Tests;
+
+public class TokenReplacementTests
+{
+    [Fact]
+    public void Token_replacer_receives_the_token_from_the_command_line_with_the_leading_at_symbol_removed()
+    {
+        var argument = new Argument<int>();
+
+        var command = new RootCommand { argument };
+
+        string receivedToken = null;
+
+        var parser = new CommandLineBuilder(command)
+                     .UseTokenReplacer((string tokenToReplace, out IReadOnlyList<string> tokens, out string message) =>
+                     {
+                         receivedToken = tokenToReplace;
+                         tokens = null;
+                         message = "oops!";
+                         return false;
+                     })
+                     .Build();
+
+        parser.Parse("@replace-me");
+
+        receivedToken.Should().Be("replace-me");
+    }
+
+    [Fact]
+    public void Token_replacer_can_expand_argument_values()
+    {
+        var argument = new Argument<int>();
+
+        var command = new RootCommand { argument };
+
+        var parser = new CommandLineBuilder(command)
+                     .UseTokenReplacer((string tokenToReplace, out IReadOnlyList<string> tokens, out string message) =>
+                     {
+                         tokens = new[] { "123" };
+                         message = null;
+                         return true;
+                     })
+                     .Build();
+
+        var result = parser.Parse("@replace-me");
+
+        result.Errors.Should().BeEmpty();
+
+        result.GetValueForArgument(argument).Should().Be(123);
+    }
+
+    [Fact]
+    public void Custom_token_replacer_can_expand_option_argument_values()
+    {
+        var option = new Option<int>("-x");
+
+        var command = new RootCommand { option };
+
+        var parser = new CommandLineBuilder(command)
+                     .UseTokenReplacer((string tokenToReplace, out IReadOnlyList<string> tokens, out string message) =>
+                     {
+                         tokens = new[] { "123" };
+                         message = null;
+                         return true;
+                     })
+                     .Build();
+
+        var result = parser.Parse("-x @replace-me");
+
+        result.Errors.Should().BeEmpty();
+
+        result.GetValueForOption(option).Should().Be(123);
+    }
+
+    [Fact]
+    public void Custom_token_replacer_can_expand_subcommands_and_options_and_argument()
+    {
+        var option = new Option<int>("-x");
+
+        var command = new RootCommand { new Command("subcommand") { option } };
+
+        var parser = new CommandLineBuilder(command)
+                     .UseTokenReplacer((string tokenToReplace, out IReadOnlyList<string> tokens, out string message) =>
+                     {
+                         tokens = new[] { "subcommand", "-x", "123" };
+                         message = null;
+                         return true;
+                     })
+                     .Build();
+
+        var result = parser.Parse("@replace-me");
+
+        result.Errors.Should().BeEmpty();
+
+        result.GetValueForOption(option).Should().Be(123);
+    }
+
+    [Fact]
+    public void Expanded_tokens_containing_whitespace_are_parsed_as_single_tokens()
+    {
+        var argument = new Argument<string>();
+
+        var command = new RootCommand { argument };
+
+        var parser = new CommandLineBuilder(command)
+                     .UseTokenReplacer((string tokenToReplace, out IReadOnlyList<string> tokens, out string message) =>
+                     {
+                         tokens = new[] { "one two three" };
+                         message = null;
+                         return true;
+                     })
+                     .Build();
+
+        var result = parser.Parse("@replace-me");
+
+        result.GetValueForArgument(argument).Should().Be("one two three");
+    }
+
+    [Fact]
+    public void Token_replacer_can_set_a_custom_error_message()
+    {
+        var argument = new Argument<string>();
+
+        var command = new RootCommand { argument };
+
+        var parser = new CommandLineBuilder(command)
+                     .UseTokenReplacer((string tokenToReplace, out IReadOnlyList<string> tokens, out string message) =>
+                     {
+                         tokens = null;
+                         message = "oops!";
+                         return false;
+                     })
+                     .Build();
+
+        var result = parser.Parse("@replace-me");
+
+        result.Errors
+              .Should()
+              .ContainSingle(e => e.Message == "oops!");
+    }
+    
+    [Fact]
+    public void When_token_replacer_returns_false_without_setting_an_error_message_then_the_command_line_is_unchanged_and_no_parse_error_is_produced()
+    {
+        var argument = new Argument<string>();
+
+        var command = new RootCommand { argument };
+
+        var parser = new CommandLineBuilder(command)
+                     .UseTokenReplacer((string tokenToReplace, out IReadOnlyList<string> tokens, out string message) =>
+                     {
+                         tokens = null;
+                         message = null;
+                         return false;
+                     })
+                     .Build();
+
+        var result = parser.Parse("@replace-me");
+
+        result.Errors.Should().BeEmpty();
+
+        result.GetValueForArgument(argument).Should().Be("@replace-me");
+    }
+
+    [Fact]
+    public void Token_replacer_will_delete_token_when_delegate_returns_true_and_sets_tokens_to_null()
+    {
+        var argument = new Argument<string[]>();
+
+        var command = new RootCommand { argument };
+
+        var parser = new CommandLineBuilder(command)
+                     .UseTokenReplacer((string tokenToReplace, out IReadOnlyList<string> tokens, out string message) =>
+                     {
+                         tokens = null;
+                         message = null;
+                         return true;
+                     })
+                     .Build();
+
+        var result = parser.Parse("@replace-me");
+
+        result.Errors.Should().BeEmpty();
+
+        result.GetValueForArgument(argument).Should().BeEmpty();
+    }
+
+    [Fact]
+    public void Token_replacer_will_delete_token_when_delegate_returns_true_and_sets_tokens_to_empty_array()
+    {
+        var argument = new Argument<string[]>();
+
+        var command = new RootCommand { argument };
+
+        var parser = new CommandLineBuilder(command)
+                     .UseTokenReplacer((string tokenToReplace, out IReadOnlyList<string> tokens, out string message) =>
+                     {
+                         tokens = Array.Empty<string>();
+                         message = null;
+                         return true;
+                     })
+                     .Build();
+
+        var result = parser.Parse("@replace-me");
+
+        result.Errors.Should().BeEmpty();
+
+        result.GetValueForArgument(argument).Should().BeEmpty();
+    }
+}
\ No newline at end of file
diff --git a/src/System.CommandLine/Parsing/StringExtensions.cs b/src/System.CommandLine/Parsing/StringExtensions.cs
index bf15c52aa9..9f5aff102d 100644
--- a/src/System.CommandLine/Parsing/StringExtensions.cs
+++ b/src/System.CommandLine/Parsing/StringExtensions.cs
@@ -137,14 +137,14 @@ configuration.TokenReplacer is { } replacer &&
                             out var newTokens,
                             out var error))
                     {
-                        argList.InsertRange(i + 1, newTokens!);
+                        argList.InsertRange(i + 1, newTokens ?? Array.Empty<string>());
+                        continue;
                     }
-                    else
+                    else if (!string.IsNullOrWhiteSpace(error))
                     {
                         errorList.Add(error!);
+                        continue;
                     }
-
-                    continue;
                 }
 
                 if (knownTokens.TryGetValue(arg, out var token))
@@ -292,8 +292,6 @@ bool PreviousTokenIsAnOptionExpectingAnArgument(out Option? option)
                 option = null;
                 return false;
             }
-
-         
         }
 
         private static List<string> NormalizeRootCommand(