[rush] Add per-iteration IPCOperationRunner persistence policy

common/changes/@microsoft/rush/bmiddha-runner-persist-policy.json

@@ -0,0 +1,9 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Add a per-iteration, host-driven runner persist policy so IPC runners can be kept hot or torn down per operation per iteration, instead of fixing persistence at plugin construction.",
+      "type": "patch"
+    }
+  ]
+}

common/reviews/api/rush-lib.api.md

@@ -653,6 +653,7 @@ export interface IOperationGraphContext extends ICreateOperationsContext {
 export interface IOperationGraphIterationOptions {
     // (undocumented)
     inputsSnapshot?: IInputsSnapshot;
+    shouldRunnerPersist?: (operation: Operation) => boolean;
     startTime?: number;
 }
 
@@ -721,6 +722,7 @@ export interface IOperationRunnerContext {
         createLogFile: boolean;
         logFileSuffix?: string;
     }): Promise<T>;
+    shouldRunnerPersist?: boolean;
     status: OperationStatus;
     stopwatch: IStopwatchResult;
 }

libraries/rush-lib/src/logic/operations/IOperationGraph.ts

@@ -21,6 +21,16 @@ export interface IOperationGraphIterationOptions {
    * The time when the iteration was scheduled, if available, as returned by `performance.now()`.
    */
   startTime?: number;
+
+  /**
+   * Returns whether the operation's runner should remain active after this iteration.
+   *
+   * @remarks
+   * When omitted, all runners remain active. Returning `false` causes a runner that supports immediate
+   * teardown to close when its operation completes. Any cold runner that remains active because its operation
+   * did not execute is closed after the iteration.
+   */
+  shouldRunnerPersist?: (operation: Operation) => boolean;
 }
 
 /**

libraries/rush-lib/src/logic/operations/IOperationRunner.ts

@@ -69,6 +69,15 @@ export interface IOperationRunnerContext {
    */
   error?: Error;
 
+  /**
+   * Whether the host wants this operation's runner to remain resident (kept warm) after the
+   * operation completes for the current iteration. Defaults to `true` when the host provides no
+   * persistence policy. Runners that own a long-lived child process (such as `IPCOperationRunner`)
+   * must release that process as soon as the operation completes when this is `false`, so that the
+   * subprocess does not consume memory while downstream operations execute.
+   */
+  shouldRunnerPersist?: boolean;
+
   /**
    * Returns a callback that invalidates this operation so that it will be re-executed in the next iteration.
    * The returned callback captures only the minimal state needed, avoiding retention of the full context.
@@ -134,6 +143,8 @@ export interface IOperationRunner {
    * If true, this runner currently owns some kind of active resource (e.g. a service or a watch process).
    * This can be used to determine if the operation is "in progress" even if it is not currently executing.
    * If the runner supports this property, it should update it as appropriate during execution.
+   * A runner that closes its resource during `executeAsync()` when `context.shouldRunnerPersist` is false
+   * must report `false` afterward so that the graph's fallback cleanup does not close it again.
    * The property is optional to avoid breaking existing implementations of IOperationRunner.
    */
   readonly isActive?: boolean;

libraries/rush-lib/src/logic/operations/IPCOperationRunner.ts

@@ -28,7 +28,6 @@ export interface IIPCOperationRunnerOptions {
   initialCommand: string;
   incrementalCommand: string | undefined;
   commandForHash: string;
-  persist: boolean;
   ignoredParameterValues: ReadonlyArray<string>;
 }
 
@@ -58,7 +57,6 @@ export class IPCOperationRunner implements IOperationRunner {
   private readonly _initialCommand: string;
   private readonly _incrementalCommand: string | undefined;
   private readonly _commandForHash: string;
-  private readonly _persist: boolean;
   private readonly _ignoredParameterValues: ReadonlyArray<string>;
 
   private _ipcProcess: ChildProcess | undefined;
@@ -72,7 +70,6 @@ export class IPCOperationRunner implements IOperationRunner {
       initialCommand,
       incrementalCommand,
       commandForHash,
-      persist,
       ignoredParameterValues
     } = options;
     this.name = name;
@@ -83,7 +80,6 @@ export class IPCOperationRunner implements IOperationRunner {
     this._incrementalCommand = incrementalCommand;
     this._commandForHash = commandForHash;
 
-    this._persist = persist;
     this._ignoredParameterValues = ignoredParameterValues;
   }
 
@@ -213,7 +209,11 @@ export class IPCOperationRunner implements IOperationRunner {
           }, reject);
         });
 
-        if (isConnected && !this._persist) {
+        // If the host does not want this runner kept warm for the next iteration, release the
+        // long-lived subprocess immediately now that the operation has completed. Deferring this to
+        // the end of the iteration would keep the subprocess resident (consuming memory) while
+        // downstream operations execute, risking RAM exhaustion.
+        if (isConnected && context.shouldRunnerPersist === false) {
           await this.closeAsync();
         }
 

libraries/rush-lib/src/logic/operations/IPCOperationRunnerPlugin.ts

@@ -81,7 +81,6 @@ export class IPCOperationRunnerPlugin implements IPhasedCommandPlugin {
             initialCommand,
             incrementalCommand,
             commandForHash,
-            persist: true,
             ignoredParameterValues
           });
 

libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts

@@ -46,6 +46,7 @@ export interface IOperationExecutionRecordContext {
   onOperationStateChanged?: (record: OperationExecutionRecord) => void;
   createEnvironment?: (record: OperationExecutionRecord) => IEnvironment;
   invalidate?: (operations: Iterable<Operation>, reason: string) => void;
+  shouldRunnerPersist?: (operation: Operation) => boolean;
   inputsSnapshot: IInputsSnapshot | undefined;
   maxParallelism: number;
 
@@ -222,6 +223,14 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera
     return this._context.createEnvironment?.(this);
   }
 
+  /**
+   * Whether this operation's runner should remain resident after the operation completes for the
+   * current iteration. Defaults to `true` (kept warm) when the host supplies no persistence policy.
+   */
+  public get shouldRunnerPersist(): boolean {
+    return this._context.shouldRunnerPersist?.(this.operation) ?? true;
+  }
+
   public getInvalidateCallback(): (reason: string) => void {
     const invalidateFn: ((operations: Iterable<Operation>, reason: string) => void) | undefined =
       this._context.invalidate;

libraries/rush-lib/src/logic/operations/OperationGraph.ts

@@ -85,6 +85,7 @@ interface IExecutionIterationContext extends IOperationExecutionRecordContext {
   promise: Promise<OperationStatus> | undefined;
 
   startTime?: number;
+  shouldRunnerPersist: IOperationGraphIterationOptions['shouldRunnerPersist'];
 
   completedOperations: number;
   totalOperations: number;
@@ -572,9 +573,16 @@ export class OperationGraph implements IOperationGraph {
   ): Promise<IExecutionIterationContext | undefined> {
     const { _getInputsSnapshotAsync: getInputsSnapshotAsync } = this;
 
-    const { startTime = performance.now(), inputsSnapshot = await getInputsSnapshotAsync?.() } =
-      iterationOptions;
-    const iterationOptionsForCallbacks: IOperationGraphIterationOptions = { startTime, inputsSnapshot };
+    const {
+      startTime = performance.now(),
+      inputsSnapshot = await getInputsSnapshotAsync?.(),
+      shouldRunnerPersist
+    } = iterationOptions;
+    const iterationOptionsForCallbacks: IOperationGraphIterationOptions = {
+      startTime,
+      inputsSnapshot,
+      shouldRunnerPersist
+    };
 
     const { hooks } = this;
 
@@ -607,6 +615,7 @@ export class OperationGraph implements IOperationGraph {
     const iterationContext: IExecutionIterationContext = {
       abortController,
       startTime,
+      shouldRunnerPersist,
       streamCollator,
       terminal,
       inputsSnapshot,
@@ -757,7 +766,8 @@ export class OperationGraph implements IOperationGraph {
 
     const iterationOptions: IOperationGraphIterationOptions = {
       inputsSnapshot: iterationContext.inputsSnapshot,
-      startTime: iterationContext.startTime
+      startTime: iterationContext.startTime,
+      shouldRunnerPersist: iterationContext.shouldRunnerPersist
     };
 
     const executionQueue: AsyncOperationQueue = new AsyncOperationQueue(
@@ -1011,6 +1021,20 @@ export class OperationGraph implements IOperationGraph {
       telemetry.log(logEntry);
     }
 
+    // IPC operations that executed this iteration release their runner immediately upon completion.
+    // This fallback closes cold runners that remain active because their operation did not execute,
+    // for example due to a cache hit, abort, blocked dependency, or disabled operation.
+    const { shouldRunnerPersist } = iterationContext;
+    if (shouldRunnerPersist) {
+      const operationsToClose: Operation[] = [];
+      for (const operation of this.operations) {
+        if (!shouldRunnerPersist(operation) && operation.runner?.isActive !== false) {
+          operationsToClose.push(operation);
+        }
+      }
+      await this.closeRunnersAsync(operationsToClose);
+    }
+
     return status;
 
     // This function is a callback because it may write to the collatedWriter before