repo sync

.github/allowed-actions.js

@@ -30,7 +30,7 @@ module.exports = [
   'rachmari/labeler@832d42ec5523f3c6d46e8168de71cd54363e3e2e',
   'repo-sync/github-sync@3832fe8e2be32372e1b3970bbae8e7079edeec88',
   'repo-sync/pull-request@33777245b1aace1a58c87a29c90321aa7a74bd7d',
-  'rtCamp/action-slack-notify@e17352feaf9aee300bf0ebc1dfbf467d80438815',
+  'someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd',
   'tjenkinson/gh-action-auto-merge-dependency-updates@cee2ac0',
   'EndBug/add-and-commit@9358097a71ad9fb9e2f9624c6098c89193d83575'
 ]

.github/workflows/repo-freeze-reminders.yml

@@ -14,11 +14,10 @@ jobs:
     if: github.repository == 'github/docs-internal'
     steps:
       - name: Send Slack notification if repo is frozen
+        uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd
         if: ${{ env.FREEZE == 'true' }}
-        uses: rtCamp/action-slack-notify@e17352feaf9aee300bf0ebc1dfbf467d80438815
-        env:
-          SLACK_WEBHOOK: ${{ secrets.DOCS_ALERTS_SLACK_WEBHOOK }}
-          SLACK_USERNAME: docs-repo-sync
-          SLACK_ICON_EMOJI: ':freezing_face:'
-          SLACK_COLOR: '#51A0D5' # Carolina Blue
-          SLACK_MESSAGE: All repo-sync runs will fail for ${{ github.repository }} because the repo is currently frozen!
+        with:
+          channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
+          bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
+          color: info
+          text: All repo-sync runs will fail for ${{ github.repository }} because the repo is currently frozen!

.github/workflows/repo-sync-stalls.yml

@@ -45,11 +45,10 @@ jobs:
               }
             })
       - name: Send Slack notification if workflow fails
-        uses: rtCamp/action-slack-notify@e17352feaf9aee300bf0ebc1dfbf467d80438815
-        if: ${{ failure() }}
-        env:
-          SLACK_WEBHOOK: ${{ secrets.DOCS_ALERTS_SLACK_WEBHOOK }}
-          SLACK_USERNAME: docs-repo-sync
-          SLACK_ICON_EMOJI: ':ohno:'
-          SLACK_COLOR: '#B90E0A' # Crimson
-          SLACK_MESSAGE: Repo sync appears to be stalled for ${{github.repository}}. See https://github.com/${{github.repository}}/pulls?q=is%3Apr+is%3Aopen+repo+sync
+        uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd
+        if: failure()
+        with:
+          channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
+          bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
+          color: failure
+          text: Repo sync appears to be stalled for ${{github.repository}}. See https://github.com/${{github.repository}}/pulls?q=is%3Apr+is%3Aopen+repo+sync

.github/workflows/repo-sync.yml

@@ -70,11 +70,10 @@ jobs:
           number: ${{ steps.find-pull-request.outputs.number }}
 
       - name: Send Slack notification if workflow fails
-        uses: rtCamp/action-slack-notify@e17352feaf9aee300bf0ebc1dfbf467d80438815
-        if: ${{ failure() }}
-        env:
-          SLACK_WEBHOOK: ${{ secrets.DOCS_ALERTS_SLACK_WEBHOOK }}
-          SLACK_USERNAME: docs-repo-sync
-          SLACK_ICON_EMOJI: ':ohno:'
-          SLACK_COLOR: '#B90E0A' # Crimson
-          SLACK_MESSAGE: The last repo-sync run for ${{github.repository}} failed. See https://github.com/${{github.repository}}/actions?query=workflow%3A%22Repo+Sync%22
+        uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd
+        if: failure()
+        with:
+          channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
+          bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
+          color: failure
+          text: The last repo-sync run for ${{github.repository}} failed. See https://github.com/${{github.repository}}/actions?query=workflow%3A%22Repo+Sync%22

.github/workflows/sync-algolia-search-indices.yml

@@ -33,8 +33,10 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: npm run sync-search
       - name: Send slack notification if workflow run fails
-        uses: rtCamp/action-slack-notify@e17352feaf9aee300bf0ebc1dfbf467d80438815
+        uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd
         if: failure()
-        env:
-          SLACK_WEBHOOK: ${{ secrets.DOCS_ALERTS_SLACK_WEBHOOK }}
-          SLACK_MESSAGE: The last Algolia workflow run for ${{github.repository}} failed. Search actions for `workflow:Algolia`
+        with:
+          channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
+          bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
+          color: failure
+          text: The last Algolia workflow run for ${{github.repository}} failed. Search actions for `workflow:Algolia`

lib/page.js

@@ -1,5 +1,5 @@
 const assert = require('assert')
-const fs = require('fs')
+const fs = require('fs').promises
 const path = require('path')
 const cheerio = require('cheerio')
 const patterns = require('./patterns')
@@ -23,15 +23,30 @@ const slash = require('slash')
 const statsd = require('./statsd')
 
 class Page {
-  constructor (opts) {
+  static async init (opts) {
     assert(opts.relativePath, 'relativePath is required')
     assert(opts.basePath, 'basePath is required')
+
+    const relativePath = slash(opts.relativePath)
+    const fullPath = slash(path.join(opts.basePath, relativePath))
+    const raw = await fs.readFile(fullPath, 'utf8')
+
+    return new Page({ ...opts, relativePath, fullPath, raw })
+  }
+
+  static async exists (path) {
+    try {
+      return await fs.stat(path)
+    } catch (err) {
+      if (err.code === 'ENOENT') return false
+      console.error(err)
+    }
+  }
+
+  constructor (opts) {
     assert(opts.languageCode, 'languageCode is required')
 
     Object.assign(this, { ...opts })
-    this.relativePath = slash(this.relativePath)
-    this.fullPath = slash(path.join(this.basePath, this.relativePath))
-    this.raw = fs.readFileSync(this.fullPath, 'utf8')
 
     // TODO remove this when crowdin-support issue 66 has been resolved
     if (this.languageCode !== 'en' && this.raw.includes(': verdadero')) {

lib/pages.js

@@ -2,42 +2,50 @@ const path = require('path')
 const walk = require('walk-sync').entries
 const Page = require('./page')
 const languages = require('./languages')
-const fs = require('fs')
+const { mapLimit, filterLimit } = require('async')
+const FILE_READ_LIMIT = 500
 
 async function loadPageList () {
   const pageList = []
 
   // load english pages
   const englishPath = path.join(__dirname, '..', languages.en.dir, 'content')
-  const englishPages = walk(englishPath)
-    .filter(({ relativePath }) => {
-      return relativePath.endsWith('.md') &&
-      !relativePath.includes('README')
-    })
-    .map(fileData => new Page({ ...fileData, languageCode: languages.en.code }))
-
+  const englishPaths = walk(englishPath)
+    .filter(({ relativePath }) =>
+      relativePath.endsWith('.md') && !relativePath.includes('README')
+    )
+  const englishPages = await mapLimit(
+    englishPaths,
+    FILE_READ_LIMIT,
+    async fileData => await Page.init({ ...fileData, languageCode: languages.en.code })
+  )
   pageList.push(...englishPages)
 
   // load matching pages in other languages
-  for (const page of englishPages) {
-    for (const language of Object.values(languages)) {
-      if (language.code === 'en') continue
-
+  let localizedPaths = Object.values(languages)
+    .filter(({ code }) => code !== 'en')
+    .map(language => {
       const basePath = path.join(__dirname, '..', language.dir, 'content')
-      const localizedPath = path.join(basePath, page.relativePath)
-      try {
-        fs.statSync(localizedPath)
-      } catch (_) {
-        continue
-      }
-
-      pageList.push(new Page({
-        relativePath: page.relativePath,
+      return englishPages.map(page => ({
         basePath,
+        relativePath: page.relativePath,
+        localizedPath: path.join(basePath, page.relativePath),
         languageCode: language.code
       }))
-    }
-  }
+    })
+    .flat()
+  localizedPaths = await filterLimit(
+    localizedPaths,
+    FILE_READ_LIMIT,
+    async ({ localizedPath }) => Page.exists(localizedPath)
+  )
+  const localizedPages = await mapLimit(
+    localizedPaths,
+    FILE_READ_LIMIT,
+    async ({ basePath, relativePath, languageCode }) =>
+      await Page.init({ basePath, relativePath, languageCode })
+  )
+  pageList.push(...localizedPages)
 
   return pageList
 }

tests/browser/browser.js

@@ -3,13 +3,17 @@ const sleep = require('await-sleep')
 const querystring = require('querystring')
 
 describe('homepage', () => {
+  jest.setTimeout(60 * 1000)
+
   test('should be titled "GitHub Documentation"', async () => {
     await page.goto('http://localhost:4001')
     await expect(page.title()).resolves.toMatch('GitHub Documentation')
   })
 })
 
 describe('algolia browser search', () => {
+  jest.setTimeout(60 * 1000)
+
   it('works on the homepage', async () => {
     await page.goto('http://localhost:4001/en')
     await page.click('#search-input-container input[type="search"]')

tests/content/crowdin-config.js

@@ -4,6 +4,8 @@ const ignoredPagePaths = config.files[0].ignore
 const ignoredDataPaths = config.files[2].ignore
 
 describe('crowdin.yml config file', () => {
+  jest.setTimeout(60 * 1000)
+
   let pages
   beforeAll(async (done) => {
     pages = await loadPages()

tests/content/site-data-references.js

@@ -7,6 +7,8 @@ const fs = require('fs').promises
 const path = require('path')
 
 describe('data references', () => {
+  jest.setTimeout(60 * 1000)
+
   let data, pages
 
   beforeAll(async (done) => {

tests/routing/redirects.js

@@ -17,17 +17,17 @@ describe('redirects', () => {
     done()
   })
 
-  test('page.redirects is an array', () => {
-    const page = new Page({
+  test('page.redirects is an array', async () => {
+    const page = await Page.init({
       relativePath: 'github/collaborating-with-issues-and-pull-requests/about-branches.md',
       basePath: path.join(__dirname, '../../content'),
       languageCode: 'en'
     })
     expect(isPlainObject(page.redirects)).toBe(true)
   })
 
-  test('dotcom homepage page.redirects', () => {
-    const page = new Page({
+  test('dotcom homepage page.redirects', async () => {
+    const page = await Page.init({
       relativePath: 'github/index.md',
       basePath: path.join(__dirname, '../../content'),
       languageCode: 'en'
@@ -41,7 +41,7 @@ describe('redirects', () => {
   })
 
   test('converts single `redirect_from` strings values into arrays', async () => {
-    const page = new Page({
+    const page = await Page.init({
       relativePath: 'github/collaborating-with-issues-and-pull-requests/about-conversations-on-github.md',
       basePath: path.join(__dirname, '../../content'),
       languageCode: 'en'

tests/unit/find-page.js

@@ -8,7 +8,7 @@ describe('find page', () => {
   jest.setTimeout(1000 * 1000)
 
   test('falls back to the English page if it can\'t find a localized page', async () => {
-    const page = new Page({
+    const page = await Page.init({
       relativePath: 'page-that-does-not-exist-in-translations-dir.md',
       basePath: path.join(__dirname, '../fixtures'),
       languageCode: 'en'
@@ -24,7 +24,7 @@ describe('find page', () => {
   })
 
   test('follows redirects', async () => {
-    const page = new Page({
+    const page = await Page.init({
       relativePath: 'page-with-redirects.md',
       basePath: path.join(__dirname, '../fixtures'),
       languageCode: 'en'

tests/unit/liquid-helpers.js

@@ -5,6 +5,8 @@ const { set } = require('lodash')
 const nonEnterpriseDefaultVersion = require('../../lib/non-enterprise-default-version')
 
 describe('liquid helper tags', () => {
+  jest.setTimeout(60 * 1000)
+
   const context = {}
   let pageMap
   beforeAll(async (done) => {