repo sync

.github/actions-scripts/staging-commit-status-success.js

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+
+import getOctokit from '../../script/helpers/github.js'
+
+const { GITHUB_TOKEN } = process.env
+
+// Exit if GitHub Actions PAT is not found
+if (!GITHUB_TOKEN) {
+  throw new Error('You must supply a GITHUB_TOKEN environment variable!')
+}
+
+// This helper uses the `GITHUB_TOKEN` implicitly!
+// We're using our usual version of Octokit vs. the provided `github`
+// instance to avoid versioning discrepancies.
+const octokit = getOctokit()
+
+const { CONTEXT_NAME, ACTIONS_RUN_LOG, HEAD_SHA } = process.env
+if (!CONTEXT_NAME) {
+  throw new Error('$CONTEXT_NAME not set')
+}
+if (!ACTIONS_RUN_LOG) {
+  throw new Error('$ACTIONS_RUN_LOG not set')
+}
+if (!HEAD_SHA) {
+  throw new Error('$HEAD_SHA not set')
+}
+
+await octokit.repos.createCommitStatus({
+  owner,
+  repo,
+  sha: HEAD_SHA,
+  context: CONTEXT_NAME,
+  state: 'success',
+  description: 'Successfully deployed! See logs.',
+  target_url: ACTIONS_RUN_LOG,
+})

.github/actions-scripts/staging-deploy.js

@@ -21,7 +21,7 @@ if (!HEROKU_API_TOKEN) {
 // instance to avoid versioning discrepancies.
 const octokit = getOctokit()
 
-const { RUN_ID, PR_URL, SOURCE_BLOB_URL, CONTEXT_NAME, ACTIONS_RUN_LOG, HEAD_SHA } = process.env
+const { RUN_ID, PR_URL, SOURCE_BLOB_URL } = process.env
 if (!RUN_ID) {
   throw new Error('$RUN_ID not set')
 }
@@ -31,15 +31,6 @@ if (!PR_URL) {
 if (!SOURCE_BLOB_URL) {
   throw new Error('$SOURCE_BLOB_URL not set')
 }
-if (!CONTEXT_NAME) {
-  throw new Error('$CONTEXT_NAME not set')
-}
-if (!ACTIONS_RUN_LOG) {
-  throw new Error('$ACTIONS_RUN_LOG not set')
-}
-if (!HEAD_SHA) {
-  throw new Error('$HEAD_SHA not set')
-}
 
 const { owner, repo, pullNumber } = parsePrUrl(PR_URL)
 if (!owner || !repo || !pullNumber) {
@@ -62,13 +53,3 @@ await deployToStaging({
   sourceBlobUrl: SOURCE_BLOB_URL,
   runId: RUN_ID,
 })
-
-await octokit.repos.createCommitStatus({
-  owner,
-  repo,
-  sha: HEAD_SHA,
-  context: CONTEXT_NAME,
-  state: 'success',
-  description: 'Successfully deployed! See logs.',
-  target_url: ACTIONS_RUN_LOG,
-})

.github/workflows/prod-build-deploy.yml

@@ -52,6 +52,12 @@ jobs:
           DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }}
           GIT_BRANCH: main
 
+      - name: Cache nextjs build
+        uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353
+        with:
+          path: .next/cache
+          key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }}
+
       - name: Build
         run: npm run build
 

.github/workflows/staging-build-and-deploy-pr.yml

@@ -0,0 +1,227 @@
+name: Staging - Build and Deploy PR (fast and private-only)
+
+# **What it does**: Builds and deploys PRs to staging but ONLY for docs-internal
+# **Why we have it**: Most PRs are made on the private repo. Let's make those extra fast if we can worry less about security.
+# **Who does it impact**: All staff.
+
+# This whole workflow is only guaranteed to be secure in the *private
+# repo* and because we repo-sync these files over the to the public one,
+# IT'S CRUCIALLY IMPORTANT THAT THIS WORKFLOW IS ONLY ENABLED IN docs-internal!
+
+on:
+  # Ideally, we'd like to use 'pull_request' because we can more easily
+  # test changes to this workflow without relying on merges to 'main'.
+  # But this is guaranteed to be safer and won't have the problem of
+  # necessary secrets not being available.
+  # Perhaps some day when we're confident this workflow will always
+  # work in a regular PR, we can switch to that.
+  pull_request_target:
+
+permissions:
+  actions: read
+  contents: read
+  deployments: write
+  pull-requests: read
+  statuses: write
+
+# This allows one Build workflow run to interrupt another
+# These are different from the concurrency in that here it checks if the
+# whole workflow runs again. The "inner concurrency" is used for
+# undeployments to cleaning up resources.
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label }}'
+  cancel-in-progress: true
+
+jobs:
+  build-and-deploy:
+    # Important. This whole file is only supposed to run in the PRIVATE repo.
+    if: ${{ github.repository == 'github/docs-internal' }}
+
+    # The assumption here is that self-hosted is faster (e.g CPU power)
+    # that the regular ones. And it matters in this workflow because
+    # we do heavy CPU stuff with `npm run build` and `tar`
+    # runs-on: ubuntu-latest
+    runs-on: self-hosted
+
+    timeout-minutes: 5
+    # This interrupts Build, Deploy, and pre-write Undeploy workflow runs in
+    # progress for this PR branch.
+    concurrency:
+      group: 'PR Staging @ ${{ github.event.pull_request.head.label }}'
+      cancel-in-progress: true
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          lfs: 'true'
+          # To prevent issues with cloning early access content later
+          persist-credentials: 'false'
+
+      - name: Check out LFS objects
+        run: git lfs checkout
+
+      - name: Setup node
+        uses: actions/setup-node@04c56d2f954f1e4c69436aa54cfef261a018f458
+        with:
+          node-version: 16.13.x
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Cache nextjs build
+        uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353
+        with:
+          path: .next/cache
+          key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }}
+
+      - name: Build
+        run: npm run build
+
+      - name: Clone early access
+        run: node script/early-access/clone-for-build.js
+        env:
+          DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }}
+          GIT_BRANCH: ${{ github.event.pull_request.head.sha }}
+
+      - name: Check that the PR isn't blocking deploys
+        # We can't use ${{...}} on this if statement because of this bug
+        # https://github.com/cschleiden/actions-linter/issues/114
+        if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'automated-block-deploy')
+        run: |
+          echo "The PR appears to have the label 'automated-block-deploy'"
+          echo "Will not proceed to deploy the PR."
+          exit 2
+
+      - name: Create a Heroku build source
+        id: build-source
+        uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d
+        env:
+          HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }}
+        with:
+          script: |
+            const { owner, repo } = context.repo
+
+            if (owner !== 'github') {
+              throw new Error(`Repository owner must be 'github' but was: ${owner}`)
+            }
+            if (repo !== 'docs-internal') {
+              throw new Error(`Repository name must be 'docs-internal' but was: ${repo}`)
+            }
+
+            const Heroku = require('heroku-client')
+            const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN })
+
+            try {
+              const { source_blob: sourceBlob } = await heroku.post('/sources')
+              const { put_url: uploadUrl, get_url: downloadUrl } = sourceBlob
+
+              core.setOutput('upload_url', uploadUrl)
+              core.setOutput('download_url', downloadUrl)
+            } catch (error) {
+              if (error.statusCode === 503) {
+                console.error('💀 Heroku may be down! Please check its Status page: https://status.heroku.com/')
+              }
+              throw error
+            }
+
+      - name: Remove development-only dependencies
+        run: npm prune --production
+
+      - name: Remove all npm scripts
+        run: npm pkg delete scripts
+
+      - name: Set npm script for Heroku build to noop
+        run: npm set-script heroku-postbuild "echo 'Application was pre-built!'"
+
+      - name: Delete heavy things we won't need deployed
+        run: |
+
+          # The dereferenced file is not used in runtime once the
+          # decorated file has been created from it.
+          rm -rf lib/rest/static/dereferenced
+
+          # Translations are never tested in Staging builds
+          # but let's keep the empty directory.
+          rm -rf translations
+          mkdir translations
+
+          # Delete all the big search indexes that are NOT English (`*-en-*`)
+          pushd lib/search/indexes
+          ls | grep -v '\-en\-' | xargs rm
+          popd
+
+          # Note! Some day it would be nice to be able to delete
+          # all the heavy assets because they bloat the tarball.
+          # But it's not obvious how to test it then. For now, we'll have
+          # to accept that every staging build has a copy of the images.
+
+          # The assumption here is that a staging build will not
+          # need these legacy redirects. Only the redirects from
+          # front-matter will be at play.
+          # These static redirects json files are notoriously large
+          # and they make the tarball unnecessarily large.
+          echo '[]' > lib/redirects/static/archived-frontmatter-fallbacks.json
+          echo '{}' > lib/redirects/static/developer.json
+          echo '{}' > lib/redirects/static/archived-redirects-from-213-to-217.json
+
+          # This will turn every `lib/**/static/*.json` into
+          # an equivalent `lib/**/static/*.json.br` file.
+          # Once the server starts, it'll know to fall back to reading
+          # the `.br` equivalent if the `.json` file isn't present.
+          node .github/actions-scripts/compress-large-files.js
+
+      - name: Make the tarball for Heroku
+        run: |
+          # We can't delete the .next/cache directory from the workflow
+          # because it's needed for caching, but we can at least exclude it
+          # from the tarball. Then it can be cached but not weigh down the
+          # tarball we intend to deploy.
+          tar -zc --exclude=.next/cache --file=app.tar.gz \
+            node_modules/ \
+            .next/ \
+            assets/ \
+            content/ \
+            data/ \
+            includes/ \
+            lib/ \
+            middleware/ \
+            translations/ \
+            server.mjs \
+            package*.json \
+            .npmrc \
+            feature-flags.json \
+            next.config.js \
+            app.json \
+            Procfile
+
+          du -sh app.tar.gz
+
+      # See: https://devcenter.heroku.com/articles/build-and-release-using-the-api#sources-endpoint
+      - name: Upload to the Heroku build source
+        env:
+          UPLOAD_URL: ${{ steps.build-source.outputs.upload_url }}
+        run: |
+          curl "$UPLOAD_URL" \
+            -X PUT \
+            -H 'Content-Type:' \
+            --data-binary @app.tar.gz
+
+      # 'npm install' is faster than 'npm ci' because it only needs to
+      # *append* what's missing from ./node_modules/
+      - name: Re-install dependencies so we get devDependencies back
+        run: npm install --no-audit --no-fund --only=dev
+
+      - name: Deploy
+        id: deploy
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }}
+          HYDRO_ENDPOINT: ${{ secrets.HYDRO_ENDPOINT }}
+          HYDRO_SECRET: ${{ secrets.HYDRO_SECRET }}
+          PR_URL: ${{ github.event.pull_request.html_url }}
+          SOURCE_BLOB_URL: ${{ steps.build-source.outputs.download_url }}
+          ALLOWED_POLLING_FAILURES_PER_PHASE: '15'
+          RUN_ID: ${{ github.run_id }}
+        run: .github/actions-scripts/staging-deploy.js

.github/workflows/staging-build-pr.yml

@@ -4,6 +4,8 @@ name: Staging - Build PR
 # **Why we have it**: Because it's not safe to share our deploy secrets with forked repos: https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
 # **Who does it impact**: All contributors.
 
+# IT'S CRUCIALLY IMPORTANT THAT THIS WORKFLOW IS ONLY ENABLED IN docs!
+
 on:
   pull_request:
     types:
@@ -31,7 +33,9 @@ concurrency:
 
 jobs:
   build-pr:
-    if: ${{ github.repository == 'github/docs-internal' || github.repository == 'github/docs' }}
+    # Important. This whole file is only supposed to run in the PUBLIC repo.
+    if: ${{ github.repository == 'github/docs' }}
+
     runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'github/docs-internal'] }}
     timeout-minutes: 5
     # This interrupts Build, Deploy, and pre-write Undeploy workflow runs in
@@ -45,7 +49,7 @@ jobs:
 
       # Make sure only approved files are changed if it's in github/docs
       - name: Check changed files
-        if: ${{ github.repository == 'github/docs' && github.event.pull_request.user.login != 'Octomerger' }}
+        if: ${{ github.event.pull_request.user.login != 'Octomerger' }}
         uses: dorny/paths-filter@eb75a1edc117d3756a18ef89958ee59f9500ba58
         id: filter
         with:
@@ -107,44 +111,6 @@ jobs:
       - name: Set npm script for Heroku build to noop
         run: npm set-script heroku-postbuild "echo 'Application was pre-built!'"
 
-      - name: Delete heavy things we won't need deployed
-        if: ${{ github.repository == 'github/docs-internal' }}
-        run: |
-
-          # The dereferenced file is not used in runtime once the
-          # decorated file has been created from it.
-          rm -fr lib/rest/static/dereferenced
-
-          # Translations are never tested in Staging builds
-          # but let's keep the empty directory.
-          rm -fr translations
-          mkdir translations
-
-          # Delete all the big search indexes that are NOT English (`*-en-*`)
-          pushd lib/search/indexes
-          ls | grep -v '\-en\-' | xargs rm
-          popd
-
-          # Note! Some day it would be nice to be able to delete
-          # all the heavy assets because they bloat the tarball.
-          # But it's not obvious how to test it then. For now, we'll have
-          # to accept that every staging build has a copy of the images.
-
-          # The assumption here is that a staging build will not
-          # need these legacy redirects. Only the redirects from
-          # front-matter will be at play.
-          # These static redirects json files are notoriously large
-          # and they make the tarball unnecessarily large.
-          echo '[]' > lib/redirects/static/archived-frontmatter-fallbacks.json
-          echo '{}' > lib/redirects/static/developer.json
-          echo '{}' > lib/redirects/static/archived-redirects-from-213-to-217.json
-
-          # This will turn every `lib/**/static/*.json` into
-          # an equivalent `lib/**/static/*.json.br` file.
-          # Once the server starts, it'll know to fall back to reading
-          # the `.br` equivalent if the `.json` file isn't present.
-          node .github/actions-scripts/compress-large-files.js
-
       - name: Create an archive
         # Only bother if this is actually a pull request
         if: ${{ github.event.pull_request.number }}