test: run tests in isolated subprocess

Author: jbedard

…acy-cli/e2e/setup/300-log-environment.ts …li/e2e/initialize/300-log-environment.tstests/legacy-cli/e2e/setup/300-log-environment.ts renamed to tests/legacy-cli/e2e/initialize/300-log-environment.ts tests/legacy-cli/e2e/setup/300-log-environment.ts renamed to tests/legacy-cli/e2e/initialize/300-log-environment.ts

@@ -1,8 +1,10 @@
-import { mkdtempSync, realpathSync } from 'fs';
+import { mkdirSync, mkdtempSync, realpathSync } from 'fs';
 import { tmpdir } from 'os';
 import { dirname, join } from 'path';
 import { getGlobalVariable, setGlobalVariable } from '../utils/env';
 
+const NPM_ROOT_PREFIX = 'npm-global';
+
 export default function () {
   const argv = getGlobalVariable('argv');
 
@@ -14,7 +16,9 @@ export default function () {
     tempRoot = argv.tmpdir;
   } else {
     tempRoot = mkdtempSync(join(realpathSync(tmpdir()), 'angular-cli-e2e-'));
+    mkdirSync(join(tempRoot, NPM_ROOT_PREFIX));
   }
   console.log(`  Using "${tempRoot}" as temporary directory for a new project.`);
   setGlobalVariable('tmp-root', tempRoot);
+  setGlobalVariable('npm-root', join(tempRoot, NPM_ROOT_PREFIX));
 }

…gacy-cli/e2e/setup/500-create-project.ts …cli/e2e/initialize/500-create-project.tstests/legacy-cli/e2e/setup/500-create-project.ts renamed to tests/legacy-cli/e2e/initialize/500-create-project.ts tests/legacy-cli/e2e/setup/500-create-project.ts renamed to tests/legacy-cli/e2e/initialize/500-create-project.ts

@@ -1,31 +1,25 @@
 import { mkdir, writeFile } from 'fs/promises';
-import { delimiter, join } from 'path';
+import { join } from 'path';
 import { getGlobalVariable } from '../utils/env';
 
 /**
  * Configure npm to use a unique sandboxed environment.
  */
 export default async function () {
   const tempRoot: string = getGlobalVariable('tmp-root');
-  const npmModulesPrefix = join(tempRoot, 'npm-global');
+  const npmModulesPrefix: string = getGlobalVariable('npm-root');
   const npmrc = join(tempRoot, '.npmrc');
 
+  // From this point onward all npm transactions use the "global" npm cache
+  // isolated within this e2e test invocation.
+
   // Configure npm to use the sandboxed npm globals and rc file
   process.env.NPM_CONFIG_USERCONFIG = npmrc;
   process.env.NPM_CONFIG_PREFIX = npmModulesPrefix;
 
-  // Ensure the custom npm global bin is first on the PATH
-  // https://docs.npmjs.com/cli/v8/configuring-npm/folders#executables
-  if (process.platform.startsWith('win')) {
-    process.env.PATH = npmModulesPrefix + delimiter + process.env.PATH;
-  } else {
-    process.env.PATH = join(npmModulesPrefix, 'bin') + delimiter + process.env.PATH;
-  }
-
   // Ensure the globals directory and npmrc file exist.
   // Configure the registry in the npmrc in addition to the environment variable.
   await writeFile(npmrc, 'registry=' + getGlobalVariable('package-registry'));
-  await mkdir(npmModulesPrefix);
 
   console.log(`  Using "${npmModulesPrefix}" as e2e test global npm cache.`);
 }

tests/legacy-cli/e2e/setup/001-create-tmp-dir.ts

@@ -1,6 +1,9 @@
 import { getGlobalVariable } from '../utils/env';
 import { exec, silentNpm } from '../utils/process';
 
+const NPM_VERSION = '7.5.6';
+const YARN_VERSION = '1.22.18';
+
 export default async function () {
   const argv = getGlobalVariable('argv');
   if (argv.noglobal) {
@@ -9,10 +12,13 @@ export default async function () {
 
   const testRegistry = getGlobalVariable('package-registry');
 
-  // Install global Angular CLI.
-  await silentNpm('install', '--global', '@angular/cli', `--registry=${testRegistry}`);
-
-  try {
-    await exec(process.platform.startsWith('win') ? 'where' : 'which', 'ng');
-  } catch {}
+  // Install global Angular CLI being tested, npm+yarn used by e2e tests.
+  await silentNpm(
+    'install',
+    '--global',
+    `--registry=${testRegistry}`,
+    '@angular/cli',
+    `npm@${NPM_VERSION}`,
+    `yarn@${YARN_VERSION}`,
+  );
 }

tests/legacy-cli/e2e/setup/002-npm-sandbox.ts

@@ -3,7 +3,7 @@ import { createProjectFromAsset } from '../../../utils/assets';
 import { expectFileSizeToBeUnder, expectFileToMatch, replaceInFile } from '../../../utils/fs';
 import { execWithEnv } from '../../../utils/process';
 
-export default async function (skipCleaning: () => void) {
+export default async function () {
   const webpackCLIBin = normalize('node_modules/.bin/webpack-cli');
 
   await createProjectFromAsset('webpack/test-app');
@@ -30,6 +30,4 @@ export default async function (skipCleaning: () => void) {
     'DISABLE_V8_COMPILE_CACHE': '1',
   });
   await expectFileToMatch('dist/app.main.js', 'AppModule');
-
-  skipCleaning();
 }

tests/legacy-cli/e2e/setup/100-global-cli.ts

@@ -1,12 +1,22 @@
-const global: { [name: string]: any } = Object.create(null);
+export const ENV_PREFIX = 'LEGACI_CLI__';
 
 export function setGlobalVariable(name: string, value: any) {
-  global[name] = value;
+  process.env[ENV_PREFIX + name] = JSON.stringify(value);
 }
 
 export function getGlobalVariable(name: string): any {
-  if (!(name in global)) {
+  const envName = ENV_PREFIX + name;
+  if (!(envName in process.env)) {
     throw new Error(`Trying to access variable "${name}" but it's not defined.`);
   }
-  return global[name];
+  return JSON.parse(process.env[envName]);
+}
+
+export function getGlobalVariablesEnv(): { [k: string]: string } {
+  return Object.keys(process.env)
+    .filter((v) => v.startsWith(ENV_PREFIX))
+    .reduce((vars, n) => {
+      vars[n] = process.env[n];
+      return vars;
+    }, {});
 }

tests/legacy-cli/e2e/tests/packages/webpack/test-app.ts

@@ -3,12 +3,14 @@ import { SpawnOptions } from 'child_process';
 import * as child_process from 'child_process';
 import { concat, defer, EMPTY, from } from 'rxjs';
 import { repeat, takeLast } from 'rxjs/operators';
-import { getGlobalVariable } from './env';
+import { getGlobalVariable, getGlobalVariablesEnv } from './env';
 import { catchError } from 'rxjs/operators';
+import { delimiter, join, resolve } from 'path';
 const treeKill = require('tree-kill');
 
 interface ExecOptions {
   silent?: boolean;
+  announce?: boolean;
   waitForMatch?: RegExp;
   env?: { [varname: string]: string };
   stdin?: string;
@@ -24,18 +26,22 @@ export type ProcessOutput = {
   stderr: string;
 };
 
-function _exec(options: ExecOptions, cmd: string, args: string[]): Promise<ProcessOutput> {
+async function _exec(options: ExecOptions, cmd: string, args: string[]): Promise<ProcessOutput> {
   // Create a separate instance to prevent unintended global changes to the color configuration
   // Create function is not defined in the typings. See: https://github.com/doowb/ansi-colors/pull/44
   const colors = (ansiColors as typeof ansiColors & { create: () => typeof ansiColors }).create();
 
+  const announce = options.announce ?? true;
+
   let stdout = '';
   let stderr = '';
   const cwd = options.cwd ?? process.cwd();
   const env = options.env;
-  console.log(
-    `==========================================================================================`,
-  );
+  if (announce) {
+    console.log(
+      `==========================================================================================`,
+    );
+  }
 
   args = args.filter((x) => x !== undefined);
   const flags = [
@@ -46,9 +52,15 @@ function _exec(options: ExecOptions, cmd: string, args: string[]): Promise<Proce
     .join(', ')
     .replace(/^(.+)$/, ' [$1]'); // Proper formatting.
 
-  console.log(colors.blue(`Running \`${cmd} ${args.map((x) => `"${x}"`).join(' ')}\`${flags}...`));
-  console.log(colors.blue(`CWD: ${cwd}`));
-  console.log(colors.blue(`ENV: ${JSON.stringify(env)}`));
+  if (announce) {
+    console.log(
+      colors.blue(`Running \`${cmd} ${args.map((x) => `"${x}"`).join(' ')}\`${flags}...`),
+    );
+    console.log(colors.blue(`CWD: ${cwd}`));
+
+    console.log(colors.blue(`ENV: ${JSON.stringify(env)}`));
+  }
+
   const spawnOptions: SpawnOptions = {
     cwd,
     ...(env ? { env } : {}),
@@ -140,7 +152,7 @@ function _exec(options: ExecOptions, cmd: string, args: string[]): Promise<Proce
   });
 }
 
-export function extractNpmEnv() {
+export function extractNpmEnv(): { [k: string]: string } {
   return Object.keys(process.env)
     .filter((v) => NPM_CONFIG_RE.test(v))
     .reduce(
@@ -154,6 +166,16 @@ export function extractNpmEnv() {
     );
 }
 
+export function getNpmSandboxEnv(): { [k: string]: string } {
+  const tempRoot: string = getGlobalVariable('tmp-root');
+  const npmModulesPrefix: string = getGlobalVariable('npm-root');
+
+  return {
+    NPM_CONFIG_USERCONFIG: join(tempRoot, '.npmrc'),
+    NPM_CONFIG_PREFIX: npmModulesPrefix,
+  };
+}
+
 export function waitForAnyProcessOutputToMatch(
   match: RegExp,
   timeout = 30000,
@@ -285,26 +307,25 @@ export function silentNpm(
       {
         silent: true,
         cwd: (options as { cwd?: string } | undefined)?.cwd,
-        env: extractNpmEnv(),
       },
       'npm',
       params,
     );
   } else {
-    return _exec({ silent: true, env: extractNpmEnv() }, 'npm', args as string[]);
+    return _exec({ silent: true }, 'npm', args as string[]);
   }
 }
 
 export function silentYarn(...args: string[]) {
-  return _exec({ silent: true, env: extractNpmEnv() }, 'yarn', args);
+  return _exec({ silent: true }, 'yarn', args);
 }
 
 export function npm(...args: string[]) {
-  return _exec({ env: extractNpmEnv() }, 'npm', args);
+  return _exec({}, 'npm', args);
 }
 
 export function node(...args: string[]) {
-  return _exec({}, 'node', args);
+  return _exec({}, process.execPath, args);
 }
 
 export function git(...args: string[]) {
@@ -314,3 +335,50 @@ export function git(...args: string[]) {
 export function silentGit(...args: string[]) {
   return _exec({ silent: true }, 'git', args);
 }
+
+/**
+ * Launch the given entry in an child process isolated to the test environment.
+ *
+ * The test environment includes the local NPM registry, isolated NPM globals,
+ * the PATH variable only referencing the local node_modules and local NPM
+ * registry (not the test runner or standard global node_modules).
+ */
+export async function launchTestProcess(entry: string, ...args: any[]) {
+  const tempRoot: string = getGlobalVariable('tmp-root');
+
+  // Extract explicit environment variables for the test process.
+  const env = {
+    ...extractNpmEnv(),
+    ...getGlobalVariablesEnv(),
+    ...getNpmSandboxEnv(),
+  };
+
+  // Modify the PATH environment variable...
+  let paths = env.PATH.split(delimiter);
+
+  // Only include paths within the sandboxed test environment or external
+  // non angular-cli paths such as /usr/bin for generic commands.
+  paths = paths.filter((p) => p.startsWith(tempRoot) || !p.includes('angular-cli'));
+
+  // Ensure the custom npm global bin is on the PATH
+  // https://docs.npmjs.com/cli/v8/configuring-npm/folders#executables
+  if (process.platform.startsWith('win')) {
+    paths.unshift(env.NPM_CONFIG_PREFIX);
+  } else {
+    paths.unshift(join(env.NPM_CONFIG_PREFIX, 'bin'));
+  }
+
+  // Add the project node modules bin to the front of the path
+  paths.unshift(join(process.cwd(), 'node_modules', '.bin'));
+
+  env.PATH = paths.join(delimiter);
+
+  return _exec(
+    {
+      env,
+      cwd: process.cwd(),
+    },
+    process.execPath,
+    [resolve(__dirname, 'run_test_process'), entry, ...args],
+  );
+}

tests/legacy-cli/e2e/utils/env.ts

@@ -5,8 +5,8 @@ import { packages } from '../../../../lib/packages';
 import { getGlobalVariable } from './env';
 import { prependToFile, readFile, replaceInFile, writeFile } from './fs';
 import { gitCommit } from './git';
-import { installWorkspacePackages } from './packages';
-import { execAndWaitForOutputToMatch, git, ng } from './process';
+import { installPackage, installWorkspacePackages } from './packages';
+import { exec, execAndWaitForOutputToMatch, git, ng } from './process';
 
 export function updateJsonFile(filePath: string, fn: (json: any) => any | void) {
   return readFile(filePath).then((tsConfigJson) => {
@@ -39,9 +39,29 @@ export async function prepareProjectForE2e(name) {
 
   console.log(`Project ${name} created... Installing packages.`);
   await installWorkspacePackages();
+  await installPackage('puppeteer');
 
   await ng('generate', 'e2e', '--related-app-name', name);
 
+  // Initialize welenium webdrivers.
+  // Often fails the first time so attempt twice if necessary.
+  const webdriverCommand = exec.bind(
+    null,
+    'node_modules/protractor/bin/webdriver-manager',
+    'update',
+    '--standalone',
+    'false',
+    '--gecko',
+    'false',
+    '--versions.chrome',
+    '101.0.4951.41',
+  );
+  try {
+    await webdriverCommand();
+  } catch (e) {
+    await webdriverCommand();
+  }
+
   await useCIChrome('e2e');
   await useCIChrome('');
 
@@ -191,7 +211,6 @@ export async function useCIChrome(projectDir: string = ''): Promise<void> {
   const protractorConf = path.join(projectDir, 'protractor.conf.js');
   const karmaConf = path.join(projectDir, 'karma.conf.js');
 
-  const chromePath = require('puppeteer').executablePath();
   const protractorPath = require.resolve('protractor');
   const webdriverUpdatePath = require.resolve('webdriver-manager/selenium/update-config.json', {
     paths: [protractorPath],
@@ -209,7 +228,7 @@ export async function useCIChrome(projectDir: string = ''): Promise<void> {
       `browserName: 'chrome',
       chromeOptions: {
         args: ['--headless'],
-        binary: String.raw\`${chromePath}\`,
+        binary: require('puppeteer').executablePath(),
       }`,
     );
     await replaceInFile(
@@ -221,7 +240,10 @@ export async function useCIChrome(projectDir: string = ''): Promise<void> {
 
   // Use Puppeteer in karma if a config is found on the project.
   if (fs.existsSync(karmaConf)) {
-    await prependToFile(karmaConf, `process.env.CHROME_BIN = String.raw\`${chromePath}\`;`);
+    await prependToFile(
+      karmaConf,
+      `process.env.CHROME_BIN = require('puppeteer').executablePath();`,
+    );
     await replaceInFile(karmaConf, `browsers: ['Chrome']`, `browsers: ['ChromeHeadless']`);
   }
 }

tests/legacy-cli/e2e/utils/process.ts

@@ -0,0 +1,3 @@
+'use strict';
+require('../../../../lib/bootstrap-local');
+require('./test_process');