[feature] Add -CustomPipeName to pwsh and Enter-PSHostProcess (PowerShell#8889)

This allows a user to start PowerShell up with the name of the named pipe that is used for cross process communication (I.e. Enter-PSHostProcess).

Author: TylerLeonhardt

.gitignore

@@ -77,6 +77,7 @@ Temporary Items
 
 # TestsResults
 TestsResults*.xml
+ParallelXUnitResults.xml
 
 # Resharper settings
 PowerShell.sln.DotSettings.user

src/Microsoft.PowerShell.ConsoleHost/host/msh/CommandLineParameterParser.cs

@@ -168,6 +168,9 @@ public override void WriteWarningLine(string message)
 
     internal class CommandLineParameterParser
     {
+        private const int MaxPipePathLengthLinux = 108;
+        private const int MaxPipePathLengthMacOS = 104;
+
         internal static string[] validParameters = {
             "version",
             "nologo",
@@ -189,7 +192,8 @@ internal class CommandLineParameterParser
             "settingsfile",
             "help",
             "workingdirectory",
-            "removeworkingdirectorytrailingcharacter"
+            "removeworkingdirectorytrailingcharacter",
+            "custompipename"
         };
 
         internal CommandLineParameterParser(PSHostUserInterface hostUI, string bannerText, string helpText)
@@ -341,6 +345,14 @@ internal bool ShowVersion
             }
         }
 
+        internal string CustomPipeName
+        {
+            get
+            {
+                return _customPipeName;
+            }
+        }
+
         internal Serialization.DataFormat OutputFormat
         {
             get
@@ -773,6 +785,33 @@ private void ParseHelper(string[] args)
 
                     _configurationName = args[i];
                 }
+                else if (MatchSwitch(switchKey, "custompipename", "cus"))
+                {
+                    ++i;
+                    if (i >= args.Length)
+                    {
+                        WriteCommandLineError(
+                            CommandLineParameterParserStrings.MissingCustomPipeNameArgument);
+                        break;
+                    }
+
+                    if (!Platform.IsWindows)
+                    {
+                        int maxNameLength = (Platform.IsLinux ? MaxPipePathLengthLinux : MaxPipePathLengthMacOS) - Path.GetTempPath().Length;
+                        if (args[i].Length > maxNameLength)
+                        {
+                            WriteCommandLineError(
+                                string.Format(
+                                    CommandLineParameterParserStrings.CustomPipeNameTooLong,
+                                    maxNameLength,
+                                    args[i],
+                                    args[i].Length));
+                            break;
+                        }
+                    }
+
+                    _customPipeName = args[i];
+                }
                 else if (MatchSwitch(switchKey, "command", "c"))
                 {
                     if (!ParseCommand(args, ref i, noexitSeen, false))
@@ -1375,6 +1414,7 @@ private bool CollectArgs(string[] args, ref int i)
         private string _helpText;
         private bool _abortStartup;
         private bool _skipUserInit;
+        private string _customPipeName;
 #if STAMODE
         // Win8: 182409 PowerShell 3.0 should run in STA mode by default
         // -sta and -mta are mutually exclusive..so tracking them using nullable boolean

src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs

@@ -1369,6 +1369,12 @@ private uint Run(CommandLineParameterParser cpp, bool isPrestartWarned)
                 }
 #endif
 
+                // If the debug pipe name was specified, create the custom IPC channel.
+                if (!string.IsNullOrEmpty(cpp.CustomPipeName))
+                {
+                    RemoteSessionNamedPipeServer.CreateCustomNamedPipeServer(cpp.CustomPipeName);
+                }
+
                 // NTRAID#Windows Out Of Band Releases-915506-2005/09/09
                 // Removed HandleUnexpectedExceptions infrastructure
 #if STAMODE
@@ -2884,16 +2890,12 @@ internal sealed class RunspaceCreationEventArgs : EventArgs
         /// <summary>
         /// Constructs RunspaceCreationEventArgs.
         /// </summary>
-        /// <param name="initialCommand"></param>
-        /// <param name="skipProfiles"></param>
-        /// <param name="staMode"></param>
-        /// <param name="configurationName"></param>
-        /// <param name="initialCommandArgs"></param>
-        internal RunspaceCreationEventArgs(string initialCommand,
-                                           bool skipProfiles,
-                                           bool staMode,
-                                           string configurationName,
-                                           Collection<CommandParameter> initialCommandArgs)
+        internal RunspaceCreationEventArgs(
+            string initialCommand,
+            bool skipProfiles,
+            bool staMode,
+            string configurationName,
+            Collection<CommandParameter> initialCommandArgs)
         {
             InitialCommand = initialCommand;
             SkipProfiles = skipProfiles;

src/Microsoft.PowerShell.ConsoleHost/resources/CommandLineParameterParserStrings.resx

@@ -186,6 +186,12 @@
   <data name="MissingConfigurationNameArgument" xml:space="preserve">
     <value>Cannot process the command because -Configuration requires an argument that is a remote endpoint configuration name. Specify this argument and try again.</value>
   </data>
+  <data name="MissingCustomPipeNameArgument" xml:space="preserve">
+    <value>Cannot process the command because -CustomPipeName requires an argument that is a name of the pipe you want to use. Specify this argument and try again.</value>
+  </data>
+  <data name="CustomPipeNameTooLong" xml:space="preserve">
+    <value>Cannot process the command because -CustomPipeName specified is too long. Pipe names on this platform can be up to {0} characters long. Your pipe name '{1}' is {2} characters.</value>
+  </data>
   <data name="MissingSettingsFileArgument" xml:space="preserve">
     <value>Cannot process the command because -SettingsFile requires an argument that is a file path.</value>
   </data>

src/Microsoft.PowerShell.ConsoleHost/resources/ManagedEntranceStrings.resx

@@ -128,7 +128,8 @@ Type 'help' to get help.</value>
     <value>Usage: pwsh[.exe] [[-File] &lt;filePath&gt; [args]]
                   [-Command { - | &lt;script-block&gt; [-args &lt;arg-array&gt;]
                                 | &lt;string&gt; [&lt;CommandParameters&gt;] } ]
-                  [-ConfigurationName &lt;string&gt;] [-EncodedCommand &lt;Base64EncodedCommand&gt;]
+                  [-ConfigurationName &lt;string&gt;] [-CustomPipeName &lt;string&gt;]
+                  [-EncodedCommand &lt;Base64EncodedCommand&gt;]
                   [-ExecutionPolicy &lt;ExecutionPolicy&gt;] [-InputFormat {Text | XML}]
                   [-Interactive] [-NoExit] [-NoLogo] [-NonInteractive] [-NoProfile]
                   [-OutputFormat {Text | XML}] [-Version] [-WindowStyle &lt;style&gt;]
@@ -174,6 +175,17 @@ All parameters are case-insensitive.</value>
 
     Example: pwsh -ConfigurationName AdminRoles
 
+-CustomPipeName
+    Specifies the name to use for an additional IPC server (named pipe) used for debugging 
+    and other cross-process communication. This offers a predictable mechanism for connecting
+    to other PowerShell instances. Typically used with the CustomPipeName parameter on Enter-PSHostProcess.
+
+    Example: 
+    # PowerShell instance 1
+    pwsh -CustomPipeName mydebugpipe
+    # PowerShell instance 2
+    Enter-PSHostProcess -CustomPipeName mydebugpipe
+
 -EncodedCommand | -e | -ec
     Accepts a base64 encoded string version of a command. Use this parameter to submit
     commands to PowerShell that require complex quotation marks or curly braces.

src/System.Management.Automation/engine/remoting/commands/EnterPSHostProcessCommand.cs

@@ -13,6 +13,7 @@
 using System.Management.Automation.Internal;
 using System.Management.Automation.Remoting;
 using System.Management.Automation.Runspaces;
+using System.Text;
 
 namespace Microsoft.PowerShell.Commands
 {
@@ -37,6 +38,7 @@ public sealed class EnterPSHostProcessCommand : PSCmdlet
         private const string ProcessParameterSet = "ProcessParameterSet";
         private const string ProcessNameParameterSet = "ProcessNameParameterSet";
         private const string ProcessIdParameterSet = "ProcessIdParameterSet";
+        private const string PipeNameParameterSet = "PipePipeNameParameterSet";
         private const string PSHostProcessInfoParameterSet = "PSHostProcessInfoParameterSet";
 
         private const string NamedPipeRunspaceName = "PSAttachRunspace";
@@ -91,6 +93,16 @@ public PSHostProcessInfo HostProcessInfo
             set;
         }
 
+        /// <summary>
+        /// Gets or sets the custom named pipe name to connect to. This is usually used in conjunction with `pwsh -CustomPipeName`.
+        /// </summary>
+        [Parameter(Mandatory = true, ParameterSetName = EnterPSHostProcessCommand.PipeNameParameterSet)]
+        public string CustomPipeName
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Optional name of AppDomain in process to enter.  If not specified then the default AppDomain is used.
         /// </summary>
@@ -129,25 +141,34 @@ protected override void EndProcessing()
             }
 
             // Check selected process for existence, and whether it hosts PowerShell.
+            Runspace namedPipeRunspace = null;
             switch (ParameterSetName)
             {
                 case ProcessIdParameterSet:
                     Process = GetProcessById(Id);
+                    VerifyProcess(Process);
+                    namedPipeRunspace = CreateNamedPipeRunspace(Process.Id, AppDomainName);
                     break;
 
                 case ProcessNameParameterSet:
                     Process = GetProcessByName(Name);
+                    VerifyProcess(Process);
+                    namedPipeRunspace = CreateNamedPipeRunspace(Process.Id, AppDomainName);
                     break;
 
                 case PSHostProcessInfoParameterSet:
                     Process = GetProcessByHostProcessInfo(HostProcessInfo);
-                    break;
-            }
+                    VerifyProcess(Process);
 
-            VerifyProcess(Process);
+                    // Create named pipe runspace for selected process and open.
+                    namedPipeRunspace = CreateNamedPipeRunspace(Process.Id, AppDomainName);
+                    break;
 
-            // Create named pipe runspace for selected process and open.
-            Runspace namedPipeRunspace = CreateNamedPipeRunspace(Process.Id, AppDomainName);
+                case PipeNameParameterSet:
+                    VerifyPipeName(CustomPipeName);
+                    namedPipeRunspace = CreateNamedPipeRunspace(CustomPipeName);
+                    break;
+            }
 
             // Set runspace prompt.  The runspace is closed on pop so we don't
             // have to reverse this change.
@@ -187,9 +208,20 @@ protected override void StopProcessing()
 
         #region Private Methods
 
+        private Runspace CreateNamedPipeRunspace(string customPipeName)
+        {
+            NamedPipeConnectionInfo connectionInfo = new NamedPipeConnectionInfo(customPipeName);
+            return CreateNamedPipeRunspace(connectionInfo);
+        }
+
         private Runspace CreateNamedPipeRunspace(int procId, string appDomainName)
         {
             NamedPipeConnectionInfo connectionInfo = new NamedPipeConnectionInfo(procId, appDomainName);
+            return CreateNamedPipeRunspace(connectionInfo);
+        }
+
+        private Runspace CreateNamedPipeRunspace(NamedPipeConnectionInfo connectionInfo)
+        {
             TypeTable typeTable = TypeTable.LoadDefaultTypeFiles();
             RemoteRunspace remoteRunspace = RunspaceFactory.CreateRunspace(connectionInfo, this.Host, typeTable) as RemoteRunspace;
             remoteRunspace.Name = NamedPipeRunspaceName;
@@ -203,20 +235,40 @@ private Runspace CreateNamedPipeRunspace(int procId, string appDomainName)
             }
             catch (RuntimeException e)
             {
-                string msgAppDomainName = (!string.IsNullOrEmpty(appDomainName)) ? appDomainName : NamedPipeUtils.DefaultAppDomainName;
-
                 // Unwrap inner exception for original error message, if any.
                 string errorMessage = (e.InnerException != null) ? (e.InnerException.Message ?? string.Empty) : string.Empty;
 
-                ThrowTerminatingError(
-                    new ErrorRecord(
-                        new RuntimeException(
-                            StringUtil.Format(RemotingErrorIdStrings.EnterPSHostProcessCannotConnectToProcess,
-                                              msgAppDomainName, procId, errorMessage),
-                            e.InnerException),
-                        "EnterPSHostProcessCannotConnectToProcess",
-                        ErrorCategory.OperationTimeout,
-                        this));
+                if (connectionInfo.CustomPipeName != null)
+                {
+                    ThrowTerminatingError(
+                        new ErrorRecord(
+                            new RuntimeException(
+                                StringUtil.Format(
+                                    RemotingErrorIdStrings.EnterPSHostProcessCannotConnectToPipe,
+                                    connectionInfo.CustomPipeName,
+                                    errorMessage),
+                                e.InnerException),
+                            "EnterPSHostProcessCannotConnectToPipe",
+                            ErrorCategory.OperationTimeout,
+                            this));
+                }
+                else
+                {
+                    string msgAppDomainName = connectionInfo.AppDomainName ?? NamedPipeUtils.DefaultAppDomainName;
+
+                    ThrowTerminatingError(
+                        new ErrorRecord(
+                            new RuntimeException(
+                                StringUtil.Format(
+                                    RemotingErrorIdStrings.EnterPSHostProcessCannotConnectToProcess,
+                                    msgAppDomainName,
+                                    connectionInfo.ProcessId,
+                                    errorMessage),
+                                e.InnerException),
+                            "EnterPSHostProcessCannotConnectToProcess",
+                            ErrorCategory.OperationTimeout,
+                            this));
+                }
             }
             finally
             {
@@ -345,6 +397,33 @@ private void VerifyProcess(Process process)
             }
         }
 
+        private void VerifyPipeName(string customPipeName)
+        {
+            // Named Pipes are represented differently on Windows vs macOS & Linux
+            var sb = new StringBuilder(customPipeName.Length);
+            if (Platform.IsWindows)
+            {
+                sb.Append(@"\\.\pipe\");
+            }
+            else
+            {
+                sb.Append(Path.GetTempPath()).Append("CoreFxPipe_");
+            }
+
+            sb.Append(customPipeName);
+
+            string pipePath = sb.ToString();
+            if (!File.Exists(pipePath))
+            {
+                ThrowTerminatingError(
+                    new ErrorRecord(
+                        new PSArgumentException(StringUtil.Format(RemotingErrorIdStrings.EnterPSHostProcessNoNamedPipeFound, customPipeName)),
+                        "EnterPSHostProcessNoNamedPipeFound",
+                        ErrorCategory.InvalidArgument,
+                        this));
+            }
+        }
+
         #endregion
     }