test: run legacy-cli e2e tests via bazel

Author: jbedard

.bazelrc

@@ -90,6 +90,10 @@ build:snapshot --workspace_status_command="yarn -s ng-dev release build-env-stam
 build:snapshot --stamp
 build:snapshot --//:enable_snapshot_repo_deps
 
+build:e2e --workspace_status_command="yarn -s ng-dev release build-env-stamp --mode=release"
+build:e2e --stamp
+test:e2e  --test_timeout=1800
+
 build:local --//:enable_package_json_tar_deps
 
 ###############################

.circleci/dynamic_config.yml

@@ -47,7 +47,9 @@ var_6: &only_pull_requests
       only:
         - /pull\/\d+/
 
-var_7: &all_e2e_subsets ['npm', 'esbuild', 'yarn']
+# All e2e test suites
+var_7_subsets: &all_e2e_subsets ['npm', 'esbuild', 'yarn']
+var_7_build_types: &all_e2e_build_types ['e2e', 'snapshot']
 
 # Executor Definitions
 # https://circleci.com/docs/2.0/reusing-config/#authoring-reusable-executors
@@ -72,6 +74,16 @@ executors:
     working_directory: ~/ng
     resource_class: large
 
+  bazel-executor:
+    parameters:
+      nodeversion:
+        type: string
+        default: *default_nodeversion
+    docker:
+      - image: cimg/node:<< parameters.nodeversion >>
+    working_directory: ~/ng
+    resource_class: 2xlarge
+
   windows-executor:
     # Same as https://circleci.com/orbs/registry/orb/circleci/windows, but named.
     working_directory: ~/ng
@@ -136,6 +148,7 @@ commands:
         type: env_var_name
         default: CIRCLE_PROJECT_REPONAME
     steps:
+      - run: sudo cp .circleci/bazel.rc /etc/bazel.bazelrc
       - run:
           name: 'Setup bazel RBE remote execution'
           command: |
@@ -288,15 +301,18 @@ jobs:
           paths:
             - dist/_*.tgz
 
-  build-bazel-e2e:
-    executor: action-executor
+  bazel-build:
+    executor: bazel-executor
     steps:
       - custom_attach_workspace
-      - run: yarn bazel build //tests/legacy-cli/...
+      - setup_bazel_rbe
+      - run:
+          name: Bazel Build Packages
+          command: yarn bazel build //...
+      - fail_fast
 
-  unit-test:
-    executor: test-executor
-    resource_class: xlarge
+  bazel-test:
+    executor: bazel-executor
     parameters:
       nodeversion:
         type: string
@@ -329,6 +345,59 @@ jobs:
                 no_output_timeout: 40m
       - fail_fast
 
+  bazel-e2e-tests:
+    executor: bazel-executor
+    parameters:
+      build_type:
+        type: enum
+        enum: *all_e2e_build_types
+        default: 'e2e'
+      subset:
+        type: enum
+        enum: *all_e2e_subsets
+        default: 'npm'
+    steps:
+      - custom_attach_workspace
+      - browser-tools/install-chrome
+      - initialize_env
+      - setup_bazel_rbe
+      - run: mkdir /mnt/ramdisk/e2e
+      - run:
+          name: Test << parameters.build_type >> << parameters.subset >>
+          command: yarn bazel test --define=E2E_TEMP=/mnt/ramdisk/e2e --config=<< parameters.build_type >> //tests/legacy-cli:e2e.<< parameters.subset >>
+          no_output_timeout: 40m
+      - store_artifacts:
+          path: dist/testlogs/tests/legacy-cli/e2e.<< parameters.subset >>
+      - store_test_results:
+          path: dist/testlogs/tests/legacy-cli/e2e.<< parameters.subset >>
+      - fail_fast
+
+  bazel-test-browsers:
+    executor: bazel-executor
+    steps:
+      - custom_attach_workspace
+      - initialize_env
+      - setup_bazel_rbe
+      - run:
+          name: Initialize Saucelabs
+          command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
+      - run:
+          name: Start Saucelabs Tunnel
+          command: ./scripts/saucelabs/start-tunnel.sh
+          background: true
+        # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests
+        # too early without Saucelabs not being ready.
+      - run: ./scripts/saucelabs/wait-for-tunnel.sh
+      - run:
+          name: E2E Saucelabs Tests
+          command: yarn bazel test --config=saucelabs //tests/legacy-cli:e2e.saucelabs
+      - run: ./scripts/saucelabs/stop-tunnel.sh
+      - store_artifacts:
+          path: dist/testlogs/tests/legacy-cli/e2e.saucelabs
+      - store_test_results:
+          path: dist/testlogs/tests/legacy-cli/e2e.saucelabs
+      - fail_fast
+
   snapshot_publish:
     executor: action-executor
     resource_class: medium
@@ -475,23 +544,65 @@ workflows:
       # These jobs only really depend on Setup, but the build job is very quick to run (~35s) and
       # will catch any build errors before proceeding to the more lengthy and resource intensive
       # Bazel jobs.
-      - unit-test:
+      - bazel-test:
           name: test-node<< matrix.nodeversion >>
           matrix:
             parameters:
               nodeversion: *all_nodeversion_major
           requires:
-            - build
-
-      # Compile the e2e tests with bazel to ensure the non-runtime typescript
-      # compilation completes succesfully.
-      - build-bazel-e2e:
-          requires:
-            - build
+            - bazel-build
 
       # Windows jobs
       - e2e-cli-win
 
+      # Bazel jobs
+      - bazel-build:
+          requires:
+            - setup
+
+      - bazel-e2e-tests:
+          name: bazel-e2e-cli-<< matrix.subset >>
+          matrix:
+            parameters:
+              subset: *all_e2e_subsets
+          build_type: 'e2e'
+          filters:
+            branches:
+              ignore:
+                - main
+                - /\d+\.\d+\.x/
+          requires:
+            - bazel-build
+
+      - bazel-e2e-tests:
+          name: bazel-e2e-snapshots-<< matrix.subset >>
+          matrix:
+            parameters:
+              subset: *all_e2e_subsets
+          build_type: 'snapshot'
+          pre-steps:
+            - when:
+                condition:
+                  and:
+                    - not:
+                        equal: [main, << pipeline.git.branch >>]
+                    - not: << pipeline.parameters.snapshot_changed >>
+                steps:
+                  # Don't run snapshot E2E's unless it's on the main branch or the snapshots file has been updated.
+                  - run: circleci-agent step halt
+          requires:
+            - bazel-build
+          filters:
+            branches:
+              only:
+                - main
+                # This is needed to run this steps on Renovate PRs that amend the snapshots package.json
+                - /^pull\/.*/
+
+      - bazel-test-browsers:
+          requires:
+            - bazel-build
+
       # Publish jobs
       - snapshot_publish:
           <<: *only_release_branches

BUILD.bazel

@@ -3,6 +3,7 @@
 # Use of this source code is governed by an MIT-style license that can be
 # found in the LICENSE file at https://angular.io/license
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
+load("@build_bazel_rules_nodejs//:index.bzl", "copy_to_bin")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -16,6 +17,14 @@ exports_files([
     "package.json",
 ])
 
+# Files required by e2e tests
+copy_to_bin(
+    name = "dot-files",
+    srcs = [
+        "package.json",
+    ],
+)
+
 # Detect if the build is running under --stamp
 config_setting(
     name = "stamp",

WORKSPACE

@@ -98,3 +98,10 @@ nodejs_register_toolchains(
     name = "node16",
     node_version = "16.13.1",
 )
+
+register_toolchains(
+    "@npm//@angular/build-tooling/bazel/git-toolchain:git_linux_toolchain",
+    "@npm//@angular/build-tooling/bazel/git-toolchain:git_macos_x86_toolchain",
+    "@npm//@angular/build-tooling/bazel/git-toolchain:git_macos_arm64_toolchain",
+    "@npm//@angular/build-tooling/bazel/git-toolchain:git_windows_toolchain",
+)

packages/angular/cli/src/utilities/version.ts

@@ -23,7 +23,7 @@ class Version {
   }
 }
 
-// TODO: Convert this to use build-time version stamping after flipping the build script to use bazel
+// TODO(bazel): Convert this to use build-time version stamping after flipping the build script to use bazel
 // export const VERSION = new Version('0.0.0-PLACEHOLDER');
 export const VERSION = new Version(
   (

tests/legacy-cli/BUILD.bazel

@@ -1,4 +1,5 @@
 load("//tools:defaults.bzl", "ts_library")
+load(":e2e.bzl", "e2e_suites")
 
 ts_library(
     name = "runner",
@@ -11,16 +12,25 @@ ts_library(
     deps = [
         "//packages/angular_devkit/core",
         "//packages/angular_devkit/core/node",
-        "//tests/legacy-cli/e2e/assets",
         "//tests/legacy-cli/e2e/utils",
         "@npm//@types/glob",
         "@npm//@types/yargs-parser",
         "@npm//ansi-colors",
         "@npm//yargs-parser",
+    ],
+)
 
+e2e_suites(
+    name = "e2e",
+    data = [
+        ":runner",
+
+        # Tests + setup
         # Loaded dynamically at runtime, not compiletime deps
+        "//tests/legacy-cli/e2e/assets",
         "//tests/legacy-cli/e2e/setup",
         "//tests/legacy-cli/e2e/initialize",
         "//tests/legacy-cli/e2e/tests",
     ],
+    runner = ":e2e_runner.ts",
 )