Prepare for compiling Babel to native ESM (#13414)

* Move dynamic import in `@babel/core` to a CJS file

* Allow compiling to ESM

* Add `USE_ESM` compile-time flag

* Fix `@babel/eslint-parser`

* Provide CJS bundle of `@babel/parser`

* Minor test fixes

* Add CommonJS proxy to `@babel/core` for backward compat

* Test ESM on CI

* TODO: skip failing packages

* Add missing `devDependency` to `@babel/helper-remap-async-to-generator`

* Update contributing.md

* Apply suggestions from self-review

* Update Gulpfile.mjs

* Prettier

* Explicitly use node in makefile

Author: nicolo-ribaudo

.eslintrc.cjs

@@ -54,7 +54,7 @@ module.exports = {
         "guard-for-in": "error",
         "import/extensions": ["error", { json: "always", cjs: "always" }],
       },
-      globals: { PACKAGE_JSON: "readonly" },
+      globals: { PACKAGE_JSON: "readonly", USE_ESM: "readonly" },
     },
     {
       files: [

.github/workflows/ci.yml

@@ -65,6 +65,30 @@ jobs:
       - name: Upload coverage report
         uses: codecov/codecov-action@v1
 
+  test-esm:
+    name: Test ESM version
+    needs: prepare-yarn-cache
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Use Node.js latest
+        uses: actions/setup-node@v3
+        with:
+          node-version: "*"
+          cache: "yarn"
+      - name: Use ESM and build
+        run: make use-esm
+        env:
+          USE_ESM: true
+      - name: Test
+        # Hack: --color has supports-color@5 returned true for GitHub CI
+        # Remove once `chalk` is bumped to 4.0.
+        run: yarn jest --ci --color
+        env:
+          BABEL_ENV: test
+          USE_ESM: true
+
   build:
     name: Build Babel Artifacts
     needs: prepare-yarn-cache

.github/workflows/e2e-tests-esm.yml

@@ -0,0 +1,82 @@
+name: E2E tests (esm)
+
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  e2e-publish:
+    name: Publish to local Verdaccio registry
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Use Node.js latest
+        uses: actions/setup-node@v3
+        with:
+          node-version: "*"
+          cache: "yarn"
+      - name: Use ESM
+        run: make use-esm
+      - name: Publish
+        run: ./scripts/integration-tests/publish-local.sh
+        env:
+          USE_ESM: true
+      - uses: actions/upload-artifact@v3
+        with:
+          name: verdaccio-workspace
+          path: /tmp/verdaccio-workspace
+
+  e2e-tests:
+    name: Test
+    needs: e2e-publish
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        project:
+          - create-react-app
+    steps:
+      - name: Get yarn1 cache directory path
+        id: yarn1-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Use Node.js latest
+        uses: actions/setup-node@v3
+        with:
+          node-version: "*"
+      - name: Get yarn3 cache directory path
+        id: yarn3-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn config get cacheFolder)"
+      - name: Use yarn1 cache
+        uses: actions/cache@v3
+        id: yarn1-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+        with:
+          path: ${{ steps.yarn1-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn1-e2e-breaking-${{ matrix.project }}-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: ${{ runner.os }}-yarn1-e2e-breaking-${{ matrix.project }}-
+      - name: Use yarn3 cache
+        uses: actions/cache@v3
+        id: yarn3-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+        with:
+          path: ${{ steps.yarn3-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn3-e2e-breaking-${{ matrix.project }}-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: ${{ runner.os }}-yarn3-e2e-breaking-${{ matrix.project }}-
+      - name: Clean babel cache
+        run: |
+          rm -rf ${{ steps.yarn1-cache-dir-path.outputs.dir }}/*babel*
+          rm -rf ${{ steps.yarn3-cache-dir-path.outputs.dir }}/*babel*
+      - uses: actions/download-artifact@v3
+        with:
+          name: verdaccio-workspace
+          path: /tmp/verdaccio-workspace
+      - name: Test
+        run: ./scripts/integration-tests/e2e-${{ matrix.project }}.sh
+        env:
+          USE_ESM: true

.gitignore

@@ -89,3 +89,5 @@ packages/babel-standalone/babel.min.js
 
 /test/runtime-integration/*/output.js
 /test/runtime-integration/*/absolute-output.js
+
+.module-type

CONTRIBUTING.md

@@ -80,6 +80,21 @@ If you wish to build a copy of Babel for distribution, then run:
 $ make build-dist
 ```
 
+## Develop compiling to CommonJS or to ECMAScript modules
+
+Babel can currently be compiled both to CJS and to ESM. You can toggle between those two
+modes by running one of the following commands:
+```sh
+make use-esm
+```
+```sh
+make use-cjs
+```
+
+Note that they need to recompile the whole monorepo, so please make sure to stop any running `make watch` process before running them.
+
+If you never run a `make use-*` (or if you delete the `.module-type` file that they generate), our build process defaults to CJS.
+
 ### Running linting/tests
 
 #### Lint
@@ -121,16 +136,17 @@ $ TEST_ONLY=babel-cli make test
   <summary>More options</summary>
   <code>TEST_ONLY</code> will also match substrings of the package name:
 
-  ```sh
-  # Run tests for the @babel/plugin-transform-classes package.
-  $ TEST_ONLY=babel-plugin-transform-classes make test
-  ```
+```sh
+# Run tests for the @babel/plugin-transform-classes package.
+$ TEST_ONLY=babel-plugin-transform-classes make test
+```
 
-  Or you can use Yarn: 
+Or you can use Yarn:
+
+```sh
+$ yarn jest babel-cli
+```
 
-  ```sh
-  $ yarn jest babel-cli
-  ```
 </details>
 <br>
 
@@ -145,16 +161,19 @@ $ TEST_GREP=transformation make test
 Substitute spaces for hyphens and forward slashes when targeting specific test names:
 
 For example, for the following path:
+
 ```sh
 packages/babel-plugin-transform-arrow-functions/test/fixtures/arrow-functions/destructuring-parameters
 ```
 
 You can use:
+
 ```sh
 $ TEST_GREP="arrow functions destructuring parameters" make test
 ```
 
 Or you can directly use Yarn:
+
 ```sh
 $ yarn jest -t "arrow functions destructuring parameters"
 ```

Gulpfile.mjs

@@ -25,6 +25,14 @@ import { resolve as importMetaResolve } from "import-meta-resolve";
 import rollupBabelSource from "./scripts/rollup-plugin-babel-source.js";
 import formatCode from "./scripts/utils/formatCode.js";
 
+let USE_ESM = false;
+try {
+  const type = fs
+    .readFileSync(new URL(".module-type", import.meta.url), "utf-8")
+    .trim();
+  USE_ESM = type === "module";
+} catch {}
+
 const require = createRequire(import.meta.url);
 const monorepoRoot = path.dirname(fileURLToPath(import.meta.url));
 
@@ -480,10 +488,15 @@ const libBundles = [
   "packages/babel-plugin-bugfix-safari-id-destructuring-collision-in-function-expression",
 ].map(src => ({
   src,
-  format: "cjs",
+  format: USE_ESM ? "esm" : "cjs",
   dest: "lib",
 }));
 
+const cjsBundles = [
+  // This is used by Prettier, @babel/register and @babel/eslint-parser
+  { src: "packages/babel-parser" },
+];
+
 const dtsBundles = ["packages/babel-types"];
 
 const standaloneBundle = [
@@ -581,6 +594,45 @@ ${fs.readFileSync(path.join(path.dirname(input), "license"), "utf8")}*/
   );
 });
 
+gulp.task("build-cjs-bundles", () => {
+  if (!USE_ESM) {
+    fancyLog(
+      chalk.yellow(
+        "Skipping CJS-compat bundles for ESM-based builds, because not compiling to ESM"
+      )
+    );
+    return Promise.resolve();
+  }
+  return Promise.all(
+    cjsBundles.map(async ({ src, external = [] }) => {
+      const input = `./${src}/lib/index.js`;
+      const output = `./${src}/lib/index.cjs`;
+
+      const bundle = await rollup({
+        input,
+        external,
+        onwarn(warning, warn) {
+          if (warning.code === "CIRCULAR_DEPENDENCY") return;
+          warn(warning);
+        },
+        plugins: [
+          rollupCommonJs({ defaultIsModuleExports: true }),
+          rollupNodeResolve({
+            extensions: [".js", ".mjs", ".cjs", ".json"],
+            preferBuiltins: true,
+          }),
+        ],
+      });
+
+      await bundle.write({
+        file: output,
+        format: "cjs",
+        sourcemap: false,
+      });
+    })
+  );
+});
+
 gulp.task(
   "build",
   gulp.series(
@@ -590,7 +642,7 @@ gulp.task(
     // rebuild @babel/types and @babel/helpers since
     // type-helpers and generated helpers may be changed
     "build-babel",
-    "generate-standalone"
+    gulp.parallel("generate-standalone", "build-cjs-bundles")
   )
 );
 
@@ -612,7 +664,8 @@ gulp.task(
       gulp.series(
         "generate-type-helpers",
         // rebuild @babel/types since type-helpers may be changed
-        "build-no-bundle"
+        "build-no-bundle",
+        "build-cjs-bundles"
       )
     )
   )
@@ -631,5 +684,11 @@ gulp.task(
       "./packages/babel-helpers/src/helpers/*.js",
       gulp.task("generate-runtime-helpers")
     );
+    if (USE_ESM) {
+      gulp.watch(
+        cjsBundles.map(({ src }) => `./${src}/lib/**.js`),
+        gulp.task("build-cjs-bundles")
+      );
+    }
   })
 );

Makefile

@@ -16,14 +16,15 @@ YARN := yarn
 NODE := $(YARN) node
 
 
-.PHONY: build build-dist watch lint fix clean test-clean test-only test test-ci publish bootstrap
+.PHONY: build build-dist watch lint fix clean test-clean test-only test test-ci publish bootstrap use-esm use-cjs
 
 build: build-no-bundle
 ifneq ("$(BABEL_COVERAGE)", "true")
 	$(MAKE) build-standalone
 endif
 
 build-bundle: clean clean-lib
+	node ./scripts/set-module-type.js
 	$(YARN) gulp build
 	$(MAKE) build-flow-typings
 	$(MAKE) build-dist
@@ -34,6 +35,7 @@ build-no-bundle-ci: bootstrap-only
 	$(MAKE) build-dist
 
 build-no-bundle: clean clean-lib
+	node ./scripts/set-module-type.js
 	BABEL_ENV=development $(YARN) gulp build-dev
 	$(MAKE) build-flow-typings
 	$(MAKE) build-dist
@@ -186,6 +188,7 @@ prepublish:
 	$(MAKE) bootstrap-only
 	$(MAKE) prepublish-build
 	IS_PUBLISH=true $(MAKE) test
+	node ./scripts/set-module-type.js clean
 
 new-version-checklist:
 	# @echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
@@ -221,6 +224,7 @@ ifneq ("$(I_AM_USING_VERDACCIO)", "I_AM_SURE")
 endif
 	$(YARN) release-tool version $(VERSION) --all --yes --tag-version-prefix="version-e2e-test-"
 	$(MAKE) prepublish-build
+	node ./scripts/set-module-type.js clean
 	YARN_NPM_PUBLISH_REGISTRY=http://localhost:4873 $(YARN) release-tool publish --yes --tag-version-prefix="version-e2e-test-"
 	$(MAKE) clean
 
@@ -230,6 +234,14 @@ bootstrap-only: clean-all
 bootstrap: bootstrap-only
 	$(MAKE) generate-tsconfig build
 
+use-cjs:
+	node ./scripts/set-module-type.js script
+	$(MAKE) bootstrap
+
+use-esm:
+	node ./scripts/set-module-type.js module
+	$(MAKE) bootstrap
+
 clean-lib:
 	$(foreach source, $(SOURCES), \
 		$(call clean-source-lib, $(source)))