From 3677b7886dcc9654390dde2515f248c59353ff9b Mon Sep 17 00:00:00 2001 From: Ryo Armanda Date: Thu, 11 Mar 2021 02:51:19 +0800 Subject: [PATCH 01/10] Track recently viewed pages for dependency changes --- packages/core/src/Site/constants.js | 1 + packages/core/src/Site/index.js | 48 +++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/packages/core/src/Site/constants.js b/packages/core/src/Site/constants.js index b6d9df5dd9..c755ad7f08 100644 --- a/packages/core/src/Site/constants.js +++ b/packages/core/src/Site/constants.js @@ -17,6 +17,7 @@ module.exports = { LAZY_LOADING_SITE_FILE_NAME: 'LazyLiveReloadLoadingSite.html', LAZY_LOADING_BUILD_TIME_RECOMMENDATION_LIMIT: 30000, LAZY_LOADING_REBUILD_TIME_RECOMMENDATION_LIMIT: 5000, + LAZY_LOADING_RECENTLY_VIEWED_PAGES_LIMIT: 5, USER_VARIABLES_PATH: '_markbind/variables.md', WIKI_SITE_NAV_PATH: '_Sidebar.md', WIKI_FOOTER_PATH: '_Footer.md', diff --git a/packages/core/src/Site/index.js b/packages/core/src/Site/index.js index 19e6c06942..895b02782d 100644 --- a/packages/core/src/Site/index.js +++ b/packages/core/src/Site/index.js @@ -55,6 +55,7 @@ const { LAZY_LOADING_SITE_FILE_NAME, LAZY_LOADING_BUILD_TIME_RECOMMENDATION_LIMIT, LAZY_LOADING_REBUILD_TIME_RECOMMENDATION_LIMIT, + LAZY_LOADING_RECENTLY_VIEWED_PAGES_LIMIT, MARKBIND_WEBSITE_URL, MAX_CONCURRENT_PAGE_GENERATION_PROMISES, PAGE_TEMPLATE_NAME, @@ -144,6 +145,7 @@ class Site { this.currentPageViewed = onePagePath ? path.resolve(this.rootPath, FsUtil.removeExtension(onePagePath)) : ''; + this.recentlyViewedPages = []; this.toRebuild = new Set(); } @@ -195,6 +197,7 @@ class Site { */ changeCurrentPage(normalizedUrl) { this.currentPageViewed = path.join(this.rootPath, normalizedUrl); + this.addToRecentlyViewedPages(this.currentPageViewed); if (this.toRebuild.has(this.currentPageViewed)) { this.beforeSiteGenerate(); @@ -205,6 +208,25 @@ class Site { return false; } + /** + * Adds the viewed page path to the front of the recently viewed pages array, while + * also maintaining the array to keep below its specified limit. + * If the viewed page is already in the array, moves it to the front. + * @param viewedPagePath The absolute path to the page, extension-less + */ + addToRecentlyViewedPages(viewedPagePath) { + const idx = this.recentlyViewedPages.indexOf(viewedPagePath); + if (idx !== -1) { + this.recentlyViewedPages.splice(idx, 1); + } + this.recentlyViewedPages.unshift(viewedPagePath); + + const sizeDiff = this.recentlyViewedPages.length - LAZY_LOADING_RECENTLY_VIEWED_PAGES_LIMIT; + if (sizeDiff > 0) { + this.recentlyViewedPages.splice(LAZY_LOADING_RECENTLY_VIEWED_PAGES_LIMIT, sizeDiff); + } + } + /** * Read and store the site config from site.json, overwrite the default base URL * if it's specified by the user. @@ -968,20 +990,26 @@ class Site { logger.warn('Rebuilding all pages as variables file was changed, or the --force-reload flag was set'); } this._setTimestampVariable(); + + let filteredIdxCounter = 0; + const recentlyViewedPagesIndexMapping = {}; const pagesToRegenerate = this.pages.filter((page) => { const doFilePathsHaveSourceFiles = filePaths.some(filePath => page.isDependency(filePath)); if (shouldRebuildAllPages || doFilePathsHaveSourceFiles) { if (this.onePagePath) { const normalizedSource = FsUtil.removeExtension(page.pageConfig.sourcePath); - const isPageBeingViewed = normalizedSource === this.currentPageViewed; + const isRecentlyViewed = this.recentlyViewedPages.some(pagePath => pagePath === normalizedSource); - if (!isPageBeingViewed) { + if (!isRecentlyViewed) { this.toRebuild.add(normalizedSource); return false; } + + recentlyViewedPagesIndexMapping[normalizedSource] = filteredIdxCounter; } + filteredIdxCounter += 1; return true; } @@ -992,6 +1020,22 @@ class Site { return; } + if (this.onePagePath) { + /* + * Reorder the regenerate queue such that recently viewed pages are prioritized + * before other pages, those of which are ordered from most-to-least recent + */ + const prioritizedPagesOrdered = []; + this.recentlyViewedPages.forEach((pagePath) => { + if (!recentlyViewedPagesIndexMapping[pagePath]) { + return; + } + const [page] = pagesToRegenerate.splice(recentlyViewedPagesIndexMapping[pagePath], 1); + prioritizedPagesOrdered.push(page); + }); + pagesToRegenerate.unshift(...prioritizedPagesOrdered); + } + logger.info(`Rebuilding ${pagesToRegenerate.length} pages`); try { From 8b53ba76b9fd82041fe6e813492af6516d6cf93b Mon Sep 17 00:00:00 2001 From: Ryo Armanda Date: Thu, 11 Mar 2021 03:25:00 +0800 Subject: [PATCH 02/10] Add documentation on tracking recently viewed pages --- docs/userGuide/cliCommands.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userGuide/cliCommands.md b/docs/userGuide/cliCommands.md index d6da128b29..527fdfd91a 100644 --- a/docs/userGuide/cliCommands.md +++ b/docs/userGuide/cliCommands.md @@ -82,7 +82,7 @@ Usage: markbind * `-o `, `--one-page `
Serves only a single page from your website **initially**. If `` is not specified, it defaults to `index.md/mbd`.
- * Thereafter, when changes to source files have been made, only the page being viewed will be rebuilt if it was affected.
+ * Thereafter, when changes to source files have been made, only the most recently viewed pages (capped to the top 5 pages) will be rebuilt if it was affected.
* Navigating to a new page will build the new page, if it has not been built before, or there were some changes to source files that affected it before navigating to it.
* {{ icon_example }} `--one-page guide/index.md` From 601987eab453cb48624c16e60b174f0e55b75db2 Mon Sep 17 00:00:00 2001 From: Ryo Armanda Date: Fri, 12 Mar 2021 21:47:16 +0800 Subject: [PATCH 03/10] Improve the recently viewed pages check on regenerate --- packages/core/src/Site/index.js | 39 ++++++++++++++------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/packages/core/src/Site/index.js b/packages/core/src/Site/index.js index 895b02782d..8a868ede26 100644 --- a/packages/core/src/Site/index.js +++ b/packages/core/src/Site/index.js @@ -991,51 +991,46 @@ class Site { } this._setTimestampVariable(); - let filteredIdxCounter = 0; - const recentlyViewedPagesIndexMapping = {}; + /* + * Note (lazy serve specific): + * The pages to regenerate are split into two arrays at first, the recently viewed pages and + * everything else. This is so that we can order the recently viewed pages first before + * being combined to everything else. + */ + const recentPagesToRegenerate = []; const pagesToRegenerate = this.pages.filter((page) => { const doFilePathsHaveSourceFiles = filePaths.some(filePath => page.isDependency(filePath)); if (shouldRebuildAllPages || doFilePathsHaveSourceFiles) { if (this.onePagePath) { const normalizedSource = FsUtil.removeExtension(page.pageConfig.sourcePath); - const isRecentlyViewed = this.recentlyViewedPages.some(pagePath => pagePath === normalizedSource); + const recentIdx = this.recentlyViewedPages.findIndex(pagePath => pagePath === normalizedSource); + const isRecentlyViewed = recentIdx !== -1; if (!isRecentlyViewed) { this.toRebuild.add(normalizedSource); - return false; + } else { + recentPagesToRegenerate[recentIdx] = page; } - recentlyViewedPagesIndexMapping[normalizedSource] = filteredIdxCounter; + return false; } - filteredIdxCounter += 1; return true; } return false; }); + + if (recentPagesToRegenerate.length) { + pagesToRegenerate.unshift(...recentPagesToRegenerate.filter(page => page)); + } + if (!pagesToRegenerate.length) { logger.info('No pages needed to be rebuilt'); return; } - if (this.onePagePath) { - /* - * Reorder the regenerate queue such that recently viewed pages are prioritized - * before other pages, those of which are ordered from most-to-least recent - */ - const prioritizedPagesOrdered = []; - this.recentlyViewedPages.forEach((pagePath) => { - if (!recentlyViewedPagesIndexMapping[pagePath]) { - return; - } - const [page] = pagesToRegenerate.splice(recentlyViewedPagesIndexMapping[pagePath], 1); - prioritizedPagesOrdered.push(page); - }); - pagesToRegenerate.unshift(...prioritizedPagesOrdered); - } - logger.info(`Rebuilding ${pagesToRegenerate.length} pages`); try { From 4247937c5cc1deefa9ee212cdb75c704c8a10575 Mon Sep 17 00:00:00 2001 From: Ryo Armanda Date: Fri, 12 Mar 2021 22:06:33 +0800 Subject: [PATCH 04/10] Add log for recently viewed pages --- packages/core/src/Site/index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/core/src/Site/index.js b/packages/core/src/Site/index.js index 8a868ede26..953cdf66a2 100644 --- a/packages/core/src/Site/index.js +++ b/packages/core/src/Site/index.js @@ -205,6 +205,11 @@ class Site { return true; } + logger.info('Recently viewed pages, from most-to-least recent:'); + this.recentlyViewedPages.forEach((pagePath, idx) => { + logger.info(`${idx + 1}. ${utils.ensurePosix(path.relative(this.rootPath, pagePath))}`); + }); + return false; } From 2b1523a7437f572822a49f4363b491d38fa6a694 Mon Sep 17 00:00:00 2001 From: Ryo Armanda Date: Sat, 13 Mar 2021 04:44:47 +0800 Subject: [PATCH 05/10] Modify page generation pipeline to support sequential generation --- packages/core/src/Site/index.js | 99 ++++++++++++++++++++++++-------- packages/core/src/utils/index.js | 18 ++++++ 2 files changed, 94 insertions(+), 23 deletions(-) diff --git a/packages/core/src/Site/index.js b/packages/core/src/Site/index.js index 953cdf66a2..71bdf0c9bb 100644 --- a/packages/core/src/Site/index.js +++ b/packages/core/src/Site/index.js @@ -905,16 +905,52 @@ class Site { } /** - * Creates the supplied pages' page generation promises at a throttled rate. - * This is done to avoid pushing too many callbacks into the event loop at once. (#1245) - * @param {Array} pages to generate - * @return {Promise} that resolves once all pages have generated + * Runs the supplied page generation tasks according to the specified mode of each task. + * A page generation task can be a sequential generation or an asynchronous generation. + * @param {Array} pageGenerationTasks Array of page generation tasks + * @return {Promise} A Promise that resolves once all pages have generated */ - generatePagesThrottled(pages) { - const progressBar = new ProgressBar(`[:bar] :current / ${pages.length} pages built`, - { total: pages.length }); + async runPageGenerationTasks(pageGenerationTasks) { + const pagesCount = pageGenerationTasks.reduce((acc, task) => acc + task.pages.length, 0); + const progressBar = new ProgressBar(`[:bar] :current / ${pagesCount} pages built`, { total: pagesCount }); progressBar.render(); + await utils.sequentialAsyncForEach(pageGenerationTasks, async (task) => { + if (task.mode === 'sequential') { + await this.generatePagesSequential(task.pages, progressBar); + } else { + await this.generatePagesAsyncThrottled(task.pages, progressBar); + } + }); + } + + /** + * Generate pages sequentially. That is, the pages are generated + * one-by-one in order. + * @param {Array} pages Pages to be generated + * @param {ProgressBar} progressBar Progress bar of the overall generation process + * @returns {Promise} A Promise that resolves once all pages have been generated + */ + async generatePagesSequential(pages, progressBar) { + await utils.sequentialAsyncForEach(pages, async (page) => { + try { + await page.generate(this.externalManager); + progressBar.tick(); + } catch (err) { + logger.error(err); + throw new Error(`Error while generating ${page.sourcePath}`); + } + }); + } + + /** + * Creates the supplied pages' page generation promises at a throttled rate. + * This is done to avoid pushing too many callbacks into the event loop at once. (#1245) + * @param {Array} pages Pages to be generated + * @param {ProgressBar} progressBar Progress bar of the overall generation process + * @return {Promise} A Promise that resolves once all pages have been generated + */ + generatePagesAsyncThrottled(pages, progressBar) { return new Promise((resolve, reject) => { const counter = { numPagesGenerated: 0 }; @@ -940,7 +976,7 @@ class Site { } /** - * Helper function for generatePagesThrottled(). + * Helper function for generatePagesAsyncThrottled(). */ static generateProgressBarStatus(progressBar, counter, pageGenerationQueue, pages, resolve) { progressBar.tick(); @@ -966,7 +1002,11 @@ class Site { this._setTimestampVariable(); this.mapAddressablePagesToPages(addressablePages, faviconUrl); - return this.generatePagesThrottled(this.pages); + const pageGenerationTask = { + mode: 'async', + pages: this.pages, + }; + return this.runPageGenerationTasks([pageGenerationTask]); } /** @@ -996,14 +1036,8 @@ class Site { } this._setTimestampVariable(); - /* - * Note (lazy serve specific): - * The pages to regenerate are split into two arrays at first, the recently viewed pages and - * everything else. This is so that we can order the recently viewed pages first before - * being combined to everything else. - */ - const recentPagesToRegenerate = []; - const pagesToRegenerate = this.pages.filter((page) => { + let recentPagesToRegenerate = []; + const asyncPagesToRegenerate = this.pages.filter((page) => { const doFilePathsHaveSourceFiles = filePaths.some(filePath => page.isDependency(filePath)); if (shouldRebuildAllPages || doFilePathsHaveSourceFiles) { @@ -1027,19 +1061,38 @@ class Site { return false; }); - if (recentPagesToRegenerate.length) { - pagesToRegenerate.unshift(...recentPagesToRegenerate.filter(page => page)); - } + /* + * As a side effect of doing assignment to an empty array, some elements might be + * undefined if it has not been assigned to anything. We filter those out here. + */ + recentPagesToRegenerate = recentPagesToRegenerate.filter(page => page); - if (!pagesToRegenerate.length) { + const totalPagesToRegenerate = recentPagesToRegenerate.length + asyncPagesToRegenerate.length; + if (totalPagesToRegenerate === 0) { logger.info('No pages needed to be rebuilt'); return; } + logger.info(`Rebuilding ${totalPagesToRegenerate} pages`); - logger.info(`Rebuilding ${pagesToRegenerate.length} pages`); + const pageGenerationTasks = []; + if (recentPagesToRegenerate.length > 0) { + const recentPagesGenerationTask = { + mode: 'sequential', + pages: recentPagesToRegenerate, + }; + pageGenerationTasks.push(recentPagesGenerationTask); + } + + if (asyncPagesToRegenerate.length > 0) { + const asyncPagesGenerationTask = { + mode: 'async', + pages: asyncPagesToRegenerate, + }; + pageGenerationTasks.push(asyncPagesGenerationTask); + } try { - await this.generatePagesThrottled(pagesToRegenerate); + await this.runPageGenerationTasks(pageGenerationTasks); await this.writeSiteData(); logger.info('Pages rebuilt'); this.calculateBuildTimeForRegenerateAffectedPages(startTime); diff --git a/packages/core/src/utils/index.js b/packages/core/src/utils/index.js index 3c4036cdd6..da1a52ebdc 100644 --- a/packages/core/src/utils/index.js +++ b/packages/core/src/utils/index.js @@ -105,4 +105,22 @@ module.exports = { return text.join('').trim(); }, + + /** + * Applies an asynchronous function for each element in an array. + * Each application evaluation is done sequentially. That is, an asynchronous function + * application of an element is evaluated only when the previous applications + * have finished. + * + * @param {Array} array The array to be iterated over + * @param func The asynchronous function to be applied to each element + * @returns {Promise} A Promise that resolves once every application has been evaluated + */ + async sequentialAsyncForEach(array, func) { + for (let i = 0; i < array.length; i += 1) { + // eslint-disable-next-line no-await-in-loop + await func(array[i]); + // eslint-enable-next-line no-await-in-loop + } + }, }; From 2c8616adc9e901dfd6ca4a00ef8f5f22f14f657e Mon Sep 17 00:00:00 2001 From: Ryo Armanda Date: Sun, 14 Mar 2021 00:25:51 +0800 Subject: [PATCH 06/10] Add countermeasure for live reload spam on recently viewed pages --- packages/cli/index.js | 21 ++++++++++++++++++++- packages/core/src/Site/constants.js | 1 + packages/core/src/Site/index.js | 18 ++++++++++-------- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/packages/cli/index.js b/packages/cli/index.js index aac678a853..5b9d7ae251 100755 --- a/packages/cli/index.js +++ b/packages/cli/index.js @@ -18,6 +18,7 @@ const { INDEX_MARKDOWN_FILE, INDEX_MARKBIND_FILE, LAZY_LOADING_SITE_FILE_NAME, + LAZY_LOADING_TIME_SINCE_LAST_REQUEST_THRESHOLD, } = require('@markbind/core/src/Site/constants'); const cliUtil = require('./src/util/cliUtil'); @@ -198,6 +199,7 @@ program } if (onePagePath) { + let prevRequestTime; const lazyReloadMiddleware = function (req, res, next) { const urlExtension = path.posix.extname(req.url); @@ -210,6 +212,23 @@ program return; } + /* + * Due to the specifics of live-server, whenever pages are , + * the server will reload all opened tabs, which causes a surge of page GET requests + * that can mess up the recently viewed pages list. This is a countermeasure of identifying + * whether a request is visited by user intent instead of live-server's reload spam or redirects. + */ + let shouldAddToRecentlyViewed; + if (!prevRequestTime) { + shouldAddToRecentlyViewed = true; + prevRequestTime = new Date(); + } else { + const requestTime = new Date(); + const timeSinceLastReq = requestTime - prevRequestTime; + shouldAddToRecentlyViewed = timeSinceLastReq >= LAZY_LOADING_TIME_SINCE_LAST_REQUEST_THRESHOLD; + prevRequestTime = requestTime; + } + if (hasNoExtension && !hasEndingSlash) { // Urls of type 'host/userGuide' - check if 'userGuide' is a raw file or does not exist const diskFilePath = path.resolve(rootFolder, req.url); @@ -227,7 +246,7 @@ program : urlWithoutBaseUrl; const urlWithoutExtension = fsUtil.removeExtension(urlWithIndex); - const didInitiateRebuild = site.changeCurrentPage(urlWithoutExtension); + const didInitiateRebuild = site.changeCurrentPage(urlWithoutExtension, shouldAddToRecentlyViewed); if (didInitiateRebuild) { req.url = utils.ensurePosix(path.join(config.baseUrl || '/', LAZY_LOADING_SITE_FILE_NAME)); } diff --git a/packages/core/src/Site/constants.js b/packages/core/src/Site/constants.js index c755ad7f08..10a59cbb5a 100644 --- a/packages/core/src/Site/constants.js +++ b/packages/core/src/Site/constants.js @@ -18,6 +18,7 @@ module.exports = { LAZY_LOADING_BUILD_TIME_RECOMMENDATION_LIMIT: 30000, LAZY_LOADING_REBUILD_TIME_RECOMMENDATION_LIMIT: 5000, LAZY_LOADING_RECENTLY_VIEWED_PAGES_LIMIT: 5, + LAZY_LOADING_TIME_SINCE_LAST_REQUEST_THRESHOLD: 75, USER_VARIABLES_PATH: '_markbind/variables.md', WIKI_SITE_NAV_PATH: '_Sidebar.md', WIKI_FOOTER_PATH: '_Footer.md', diff --git a/packages/core/src/Site/index.js b/packages/core/src/Site/index.js index 71bdf0c9bb..af7ef18fe8 100644 --- a/packages/core/src/Site/index.js +++ b/packages/core/src/Site/index.js @@ -145,7 +145,7 @@ class Site { this.currentPageViewed = onePagePath ? path.resolve(this.rootPath, FsUtil.removeExtension(onePagePath)) : ''; - this.recentlyViewedPages = []; + this.recentlyViewedPages = onePagePath ? [this.currentPageViewed] : []; this.toRebuild = new Set(); } @@ -193,11 +193,18 @@ class Site { /** * Changes the site variable of the current page being viewed, building it if necessary. * @param normalizedUrl BaseUrl-less and extension-less url of the page + * @param shouldAddToRecentlyViewed Flag on whether the page should be added to recently viewed pages list * @return Boolean of whether the page needed to be rebuilt */ - changeCurrentPage(normalizedUrl) { + changeCurrentPage(normalizedUrl, shouldAddToRecentlyViewed) { this.currentPageViewed = path.join(this.rootPath, normalizedUrl); - this.addToRecentlyViewedPages(this.currentPageViewed); + if (shouldAddToRecentlyViewed) { + this.addToRecentlyViewedPages(this.currentPageViewed); + logger.info('Recently viewed pages, from most-to-least recent:'); + this.recentlyViewedPages.forEach((pagePath, idx) => { + logger.info(`${idx + 1}. ${utils.ensurePosix(path.relative(this.rootPath, pagePath))}`); + }); + } if (this.toRebuild.has(this.currentPageViewed)) { this.beforeSiteGenerate(); @@ -205,11 +212,6 @@ class Site { return true; } - logger.info('Recently viewed pages, from most-to-least recent:'); - this.recentlyViewedPages.forEach((pagePath, idx) => { - logger.info(`${idx + 1}. ${utils.ensurePosix(path.relative(this.rootPath, pagePath))}`); - }); - return false; } From b0262034c089f0798c25306d728e407700671bbe Mon Sep 17 00:00:00 2001 From: Ryo Armanda Date: Thu, 18 Mar 2021 13:07:52 +0800 Subject: [PATCH 07/10] Change strategy of determining recently viewed pages by patching live-server --- .eslintignore | 3 + docs/userGuide/cliCommands.md | 2 +- packages/cli/index.js | 41 +- packages/cli/package.json | 2 +- packages/cli/src/lib/live-server/index.js | 456 ++++++++++++++++++++++ packages/core/src/Site/constants.js | 2 - packages/core/src/Site/index.js | 55 ++- 7 files changed, 504 insertions(+), 57 deletions(-) create mode 100644 packages/cli/src/lib/live-server/index.js diff --git a/.eslintignore b/.eslintignore index 3f8aa85e46..f46a3257d5 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,5 +1,8 @@ *.min.* node_modules + +packages/cli/src/lib/live-server/* + packages/core/src/lib/markdown-it/patches/* packages/core/src/lib/markdown-it/plugins/* !packages/core/src/lib/markdown-it/plugins/markdown-it-icons.js diff --git a/docs/userGuide/cliCommands.md b/docs/userGuide/cliCommands.md index 527fdfd91a..e297728ca5 100644 --- a/docs/userGuide/cliCommands.md +++ b/docs/userGuide/cliCommands.md @@ -82,7 +82,7 @@ Usage: markbind * `-o `, `--one-page `
Serves only a single page from your website **initially**. If `` is not specified, it defaults to `index.md/mbd`.
- * Thereafter, when changes to source files have been made, only the most recently viewed pages (capped to the top 5 pages) will be rebuilt if it was affected.
+ * Thereafter, when changes to source files have been made, the opened pages will be rebuilt if it was affected.
* Navigating to a new page will build the new page, if it has not been built before, or there were some changes to source files that affected it before navigating to it.
* {{ icon_example }} `--one-page guide/index.md` diff --git a/packages/cli/index.js b/packages/cli/index.js index 5b9d7ae251..518de9e49e 100755 --- a/packages/cli/index.js +++ b/packages/cli/index.js @@ -3,7 +3,6 @@ // Entry file for Markbind project const chokidar = require('chokidar'); const fs = require('fs-extra'); -const liveServer = require('live-server'); const path = require('path'); const program = require('commander'); const Promise = require('bluebird'); @@ -18,9 +17,9 @@ const { INDEX_MARKDOWN_FILE, INDEX_MARKBIND_FILE, LAZY_LOADING_SITE_FILE_NAME, - LAZY_LOADING_TIME_SINCE_LAST_REQUEST_THRESHOLD, } = require('@markbind/core/src/Site/constants'); +const liveServer = require('./src/lib/live-server'); const cliUtil = require('./src/util/cliUtil'); const logger = require('./src/util/logger'); @@ -140,6 +139,9 @@ program const addHandler = (filePath) => { logger.info(`[${new Date().toLocaleTimeString()}] Reload for file add: ${filePath}`); + logger.info('Synchronizing opened pages list before reload'); + const normalizedActiveUrls = liveServer.getActiveUrls().map(url => fsUtil.removeExtension(url)); + site.changeCurrentOpenedPages(normalizedActiveUrls); Promise.resolve('').then(() => { if (site.isFilepathAPage(filePath) || site.isDependencyOfPage(filePath)) { return site.rebuildSourceFiles(filePath); @@ -152,6 +154,9 @@ program const changeHandler = (filePath) => { logger.info(`[${new Date().toLocaleTimeString()}] Reload for file change: ${filePath}`); + logger.info('Synchronizing opened pages list before reload'); + const normalizedActiveUrls = liveServer.getActiveUrls().map(url => fsUtil.removeExtension(url)); + site.changeCurrentOpenedPages(normalizedActiveUrls); Promise.resolve('').then(() => { if (site.isDependencyOfPage(filePath)) { return site.rebuildAffectedSourceFiles(filePath); @@ -164,6 +169,9 @@ program const removeHandler = (filePath) => { logger.info(`[${new Date().toLocaleTimeString()}] Reload for file deletion: ${filePath}`); + logger.info('Synchronizing opened pages list before reload'); + const normalizedActiveUrls = liveServer.getActiveUrls().map(url => fsUtil.removeExtension(url)); + site.changeCurrentOpenedPages(normalizedActiveUrls); Promise.resolve('').then(() => { if (site.isFilepathAPage(filePath) || site.isDependencyOfPage(filePath)) { return site.rebuildSourceFiles(filePath); @@ -199,7 +207,6 @@ program } if (onePagePath) { - let prevRequestTime; const lazyReloadMiddleware = function (req, res, next) { const urlExtension = path.posix.extname(req.url); @@ -212,23 +219,6 @@ program return; } - /* - * Due to the specifics of live-server, whenever pages are , - * the server will reload all opened tabs, which causes a surge of page GET requests - * that can mess up the recently viewed pages list. This is a countermeasure of identifying - * whether a request is visited by user intent instead of live-server's reload spam or redirects. - */ - let shouldAddToRecentlyViewed; - if (!prevRequestTime) { - shouldAddToRecentlyViewed = true; - prevRequestTime = new Date(); - } else { - const requestTime = new Date(); - const timeSinceLastReq = requestTime - prevRequestTime; - shouldAddToRecentlyViewed = timeSinceLastReq >= LAZY_LOADING_TIME_SINCE_LAST_REQUEST_THRESHOLD; - prevRequestTime = requestTime; - } - if (hasNoExtension && !hasEndingSlash) { // Urls of type 'host/userGuide' - check if 'userGuide' is a raw file or does not exist const diskFilePath = path.resolve(rootFolder, req.url); @@ -246,7 +236,16 @@ program : urlWithoutBaseUrl; const urlWithoutExtension = fsUtil.removeExtension(urlWithIndex); - const didInitiateRebuild = site.changeCurrentPage(urlWithoutExtension, shouldAddToRecentlyViewed); + res.on('close', () => { + if (!liveServer.isLiveReloadRequest(req.originalUrl)) { + logger.info(`Opening ${fsUtil.removeExtensionPosix(req.originalUrl)}`); + const normalizedActiveUrls = liveServer.getActiveUrls() + .map(url => fsUtil.removeExtension(url)); + site.changeCurrentOpenedPages(normalizedActiveUrls); + } + }); + + const didInitiateRebuild = site.changeCurrentPage(urlWithoutExtension); if (didInitiateRebuild) { req.url = utils.ensurePosix(path.join(config.baseUrl || '/', LAZY_LOADING_SITE_FILE_NAME)); } diff --git a/packages/cli/package.json b/packages/cli/package.json index d27af5b5a3..bd9a69ebef 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -35,7 +35,7 @@ "figlet": "^1.2.4", "find-up": "^4.1.0", "fs-extra": "^9.0.1", - "live-server": "^1.2.1", + "live-server": "1.2.1", "lodash": "^4.17.15", "winston": "^2.4.4", "winston-daily-rotate-file": "^3.10.0" diff --git a/packages/cli/src/lib/live-server/index.js b/packages/cli/src/lib/live-server/index.js new file mode 100644 index 0000000000..706e902a36 --- /dev/null +++ b/packages/cli/src/lib/live-server/index.js @@ -0,0 +1,456 @@ +#!/usr/bin/env node + +/* + * Patch for live-server to expose websocket clients for external use in order to keep track + * of opened tabs. + * + * live-server locally keeps track of opened client websockets in order for it to be able + * to perform live reload whenever there are changes in the watched directory. However, the + * clients list is stored internally. + * + * This patch allows us to gain access to the information that can be gathered with the client + * websockets, which in turn can enables the support for multiple-tab development. + * + * Patch is written against live-server v1.2.1 + * The **only** changes are prefaced with a CHANGED comment + */ + +var fs = require('fs'), + connect = require('connect'), + serveIndex = require('serve-index'), + logger = require('morgan'), + WebSocket = require('faye-websocket'), + path = require('path'), + url = require('url'), + http = require('http'), + send = require('send'), + open = require('opn'), + es = require("event-stream"), + os = require('os'), + chokidar = require('chokidar'); +require('colors'); + +// CHANGED: added absolute path that directs to the live-server directory +const pathToLiveServerDir = path.dirname(require.resolve('live-server')); + +// CHANGED: correctly resolve to the live-server directory +var INJECTED_CODE = fs.readFileSync(path.join(pathToLiveServerDir, "injected.html"), "utf8"); + +// CHANGED: added active tabs property +var LiveServer = { + server: null, + watcher: null, + logLevel: 2, + activeTabs: [] +}; + +function escape(html){ + return String(html) + .replace(/&(?!\w+;)/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +// Based on connect.static(), but streamlined and with added code injecter +function staticServer(root) { + var isFile = false; + try { // For supporting mounting files instead of just directories + isFile = fs.statSync(root).isFile(); + } catch (e) { + if (e.code !== "ENOENT") throw e; + } + return function(req, res, next) { + if (req.method !== "GET" && req.method !== "HEAD") return next(); + var reqpath = isFile ? "" : url.parse(req.url).pathname; + var hasNoOrigin = !req.headers.origin; + var injectCandidates = [ new RegExp("", "i"), new RegExp(""), new RegExp("", "i")]; + var injectTag = null; + + function directory() { + var pathname = url.parse(req.originalUrl).pathname; + res.statusCode = 301; + res.setHeader('Location', pathname + '/'); + res.end('Redirecting to ' + escape(pathname) + '/'); + } + + function file(filepath /*, stat*/) { + var x = path.extname(filepath).toLocaleLowerCase(), match, + possibleExtensions = [ "", ".html", ".htm", ".xhtml", ".php", ".svg" ]; + if (hasNoOrigin && (possibleExtensions.indexOf(x) > -1)) { + // TODO: Sync file read here is not nice, but we need to determine if the html should be injected or not + var contents = fs.readFileSync(filepath, "utf8"); + for (var i = 0; i < injectCandidates.length; ++i) { + match = injectCandidates[i].exec(contents); + if (match) { + injectTag = match[0]; + break; + } + } + + // CHANGED: Added line to create new entry on non-live-reload requests + const reqUrl = req.originalUrl; + if (!LiveServer.isLiveReloadRequest(reqUrl)) { + const tabEntry = { + url: reqUrl, + client: undefined, + prevClient: undefined, + isReloading: false, + } + LiveServer.activeTabs.unshift(tabEntry); + } + + if (injectTag === null && LiveServer.logLevel >= 3) { + console.warn("Failed to inject refresh script!".yellow, + "Couldn't find any of the tags ", injectCandidates, "from", filepath); + } + } + } + + function error(err) { + if (err.status === 404) return next(); + next(err); + } + + function inject(stream) { + if (injectTag) { + // We need to modify the length given to browser + var len = INJECTED_CODE.length + res.getHeader('Content-Length'); + res.setHeader('Content-Length', len); + var originalPipe = stream.pipe; + stream.pipe = function(resp) { + originalPipe.call(stream, es.replace(new RegExp(injectTag, "i"), INJECTED_CODE + injectTag)).pipe(resp); + }; + } + } + + send(req, reqpath, { root: root }) + .on('error', error) + .on('directory', directory) + .on('file', file) + .on('stream', inject) + .pipe(res); + }; +} + +/** + * Rewrite request URL and pass it back to the static handler. + * @param staticHandler {function} Next handler + * @param file {string} Path to the entry point file + */ +function entryPoint(staticHandler, file) { + if (!file) return function(req, res, next) { next(); }; + + return function(req, res, next) { + req.url = "/" + file; + staticHandler(req, res, next); + }; +} + +/** + * Start a live server with parameters given as an object + * @param host {string} Address to bind to (default: 0.0.0.0) + * @param port {number} Port number (default: 8080) + * @param root {string} Path to root directory (default: cwd) + * @param watch {array} Paths to exclusively watch for changes + * @param ignore {array} Paths to ignore when watching files for changes + * @param ignorePattern {regexp} Ignore files by RegExp + * @param noCssInject Don't inject CSS changes, just reload as with any other file change + * @param open {(string|string[])} Subpath(s) to open in browser, use false to suppress launch (default: server root) + * @param mount {array} Mount directories onto a route, e.g. [['/components', './node_modules']]. + * @param logLevel {number} 0 = errors only, 1 = some, 2 = lots + * @param file {string} Path to the entry point file + * @param wait {number} Server will wait for all changes, before reloading + * @param htpasswd {string} Path to htpasswd file to enable HTTP Basic authentication + * @param middleware {array} Append middleware to stack, e.g. [function(req, res, next) { next(); }]. + */ +LiveServer.start = function(options) { + options = options || {}; + var host = options.host || '0.0.0.0'; + var port = options.port !== undefined ? options.port : 8080; // 0 means random + var root = options.root || process.cwd(); + var mount = options.mount || []; + var watchPaths = options.watch || [root]; + LiveServer.logLevel = options.logLevel === undefined ? 2 : options.logLevel; + var openPath = (options.open === undefined || options.open === true) ? + "" : ((options.open === null || options.open === false) ? null : options.open); + if (options.noBrowser) openPath = null; // Backwards compatibility with 0.7.0 + var file = options.file; + var staticServerHandler = staticServer(root); + var wait = options.wait === undefined ? 100 : options.wait; + var browser = options.browser || null; + var htpasswd = options.htpasswd || null; + var cors = options.cors || false; + var https = options.https || null; + var proxy = options.proxy || []; + var middleware = options.middleware || []; + var noCssInject = options.noCssInject; + var httpsModule = options.httpsModule; + + if (httpsModule) { + try { + require.resolve(httpsModule); + } catch (e) { + console.error(("HTTPS module \"" + httpsModule + "\" you've provided was not found.").red); + console.error("Did you do", "\"npm install " + httpsModule + "\"?"); + return; + } + } else { + httpsModule = "https"; + } + + // Setup a web server + var app = connect(); + + // Add logger. Level 2 logs only errors + if (LiveServer.logLevel === 2) { + app.use(logger('dev', { + skip: function (req, res) { return res.statusCode < 400; } + })); + // Level 2 or above logs all requests + } else if (LiveServer.logLevel > 2) { + app.use(logger('dev')); + } + if (options.spa) { + middleware.push("spa"); + } + // Add middleware + middleware.map(function(mw) { + if (typeof mw === "string") { + var ext = path.extname(mw).toLocaleLowerCase(); + if (ext !== ".js") { + // CHANGED: correctly resolve to the live-server directory + mw = require(path.join(pathToLiveServerDir, "middleware", mw + ".js")); + } else { + mw = require(mw); + } + } + app.use(mw); + }); + + // Use http-auth if configured + if (htpasswd !== null) { + var auth = require('http-auth'); + var basic = auth.basic({ + realm: "Please authorize", + file: htpasswd + }); + app.use(auth.connect(basic)); + } + if (cors) { + app.use(require("cors")({ + origin: true, // reflecting request origin + credentials: true // allowing requests with credentials + })); + } + mount.forEach(function(mountRule) { + var mountPath = path.resolve(process.cwd(), mountRule[1]); + if (!options.watch) // Auto add mount paths to wathing but only if exclusive path option is not given + watchPaths.push(mountPath); + app.use(mountRule[0], staticServer(mountPath)); + if (LiveServer.logLevel >= 1) + console.log('Mapping %s to "%s"', mountRule[0], mountPath); + }); + proxy.forEach(function(proxyRule) { + var proxyOpts = url.parse(proxyRule[1]); + proxyOpts.via = true; + proxyOpts.preserveHost = true; + app.use(proxyRule[0], require('proxy-middleware')(proxyOpts)); + if (LiveServer.logLevel >= 1) + console.log('Mapping %s to "%s"', proxyRule[0], proxyRule[1]); + }); + app.use(staticServerHandler) // Custom static server + .use(entryPoint(staticServerHandler, file)) + .use(serveIndex(root, { icons: true })); + + var server, protocol; + if (https !== null) { + var httpsConfig = https; + if (typeof https === "string") { + httpsConfig = require(path.resolve(process.cwd(), https)); + } + server = require(httpsModule).createServer(httpsConfig, app); + protocol = "https"; + } else { + server = http.createServer(app); + protocol = "http"; + } + + // Handle server startup errors + server.addListener('error', function(e) { + if (e.code === 'EADDRINUSE') { + var serveURL = protocol + '://' + host + ':' + port; + console.log('%s is already in use. Trying another port.'.yellow, serveURL); + setTimeout(function() { + server.listen(0, host); + }, 1000); + } else { + console.error(e.toString().red); + LiveServer.shutdown(); + } + }); + + // Handle successful server + server.addListener('listening', function(/*e*/) { + LiveServer.server = server; + + var address = server.address(); + var serveHost = address.address === "0.0.0.0" ? "127.0.0.1" : address.address; + var openHost = host === "0.0.0.0" ? "127.0.0.1" : host; + + var serveURL = protocol + '://' + serveHost + ':' + address.port; + var openURL = protocol + '://' + openHost + ':' + address.port; + + var serveURLs = [ serveURL ]; + if (LiveServer.logLevel > 2 && address.address === "0.0.0.0") { + var ifaces = os.networkInterfaces(); + serveURLs = Object.keys(ifaces) + .map(function(iface) { + return ifaces[iface]; + }) + // flatten address data, use only IPv4 + .reduce(function(data, addresses) { + addresses.filter(function(addr) { + return addr.family === "IPv4"; + }).forEach(function(addr) { + data.push(addr); + }); + return data; + }, []) + .map(function(addr) { + return protocol + "://" + addr.address + ":" + address.port; + }); + } + + // Output + if (LiveServer.logLevel >= 1) { + if (serveURL === openURL) + if (serveURLs.length === 1) { + console.log(("Serving \"%s\" at %s").green, root, serveURLs[0]); + } else { + console.log(("Serving \"%s\" at\n\t%s").green, root, serveURLs.join("\n\t")); + } + else + console.log(("Serving \"%s\" at %s (%s)").green, root, openURL, serveURL); + } + + // Launch browser + if (openPath !== null) + if (typeof openPath === "object") { + openPath.forEach(function(p) { + open(openURL + p, {app: browser}); + }); + } else { + open(openURL + openPath, {app: browser}); + } + }); + + // Setup server to listen at port + server.listen(port, host); + + // WebSocket + // CHANGED: Removed local clients variable in favour of the clients in active tabs entries + server.addListener('upgrade', function(request, socket, head) { + var ws = new WebSocket(request, socket, head); + ws.onopen = function() { ws.send('connected'); }; + + if (wait > 0) { + (function() { + var wssend = ws.send; + var waitTimeout; + ws.send = function() { + var args = arguments; + if (waitTimeout) clearTimeout(waitTimeout); + waitTimeout = setTimeout(function(){ + wssend.apply(ws, args); + }, wait); + }; + })(); + } + + ws.onclose = function() { + /* + * CHANGED: Modified to remove the active tab that has the closed socket as + * its current client on socket close. In other words, only socket close event that + * does not come from live reload will remove the active tab. + */ + LiveServer.activeTabs = LiveServer.activeTabs.filter(tab => tab.client !== ws); + }; + + // CHANGED: Added line to record tab client socket on creation + const reqUrl = path.dirname(request.url); + const tab = LiveServer.activeTabs.find(tab => tab.url === reqUrl && !tab.client); + tab.client = ws; + tab.isReloading = false; + }); + + var ignored = [ + function(testPath) { // Always ignore dotfiles (important e.g. because editor hidden temp files) + return testPath !== "." && /(^[.#]|(?:__|~)$)/.test(path.basename(testPath)); + } + ]; + if (options.ignore) { + ignored = ignored.concat(options.ignore); + } + if (options.ignorePattern) { + ignored.push(options.ignorePattern); + } + // Setup file watcher + LiveServer.watcher = chokidar.watch(watchPaths, { + ignored: ignored, + ignoreInitial: true + }); + function handleChange(changePath) { + var cssChange = path.extname(changePath) === ".css" && !noCssInject; + if (LiveServer.logLevel >= 1) { + if (cssChange) + console.log("CSS change detected".magenta, changePath); + else console.log("Change detected".cyan, changePath); + } + + // CHANGED: Prepare tab entry data before issuing reload + LiveServer.activeTabs.forEach(tab => { + if (tab.client) { + const client = tab.client; + tab.client = undefined; + tab.prevClient = client; + tab.isReloading = true; + client.send(cssChange ? 'refreshcss' : 'reload'); + } + }); + } + LiveServer.watcher + .on("change", handleChange) + .on("add", handleChange) + .on("unlink", handleChange) + .on("addDir", handleChange) + .on("unlinkDir", handleChange) + .on("ready", function () { + if (LiveServer.logLevel >= 1) + console.log("Ready for changes".cyan); + }) + .on("error", function (err) { + console.log("ERROR:".red, err); + }); + + return server; +}; + +LiveServer.shutdown = function() { + var watcher = LiveServer.watcher; + if (watcher) { + watcher.close(); + } + var server = LiveServer.server; + if (server) + server.close(); +}; + +// CHANGED: Added method to retrieve current active urls +LiveServer.getActiveUrls = () => LiveServer.activeTabs.filter(tab => !tab.isReloading).map(tab => tab.url); + +// CHANGED: Added method to check whether a request is a product of the live reload mechanism +LiveServer.isLiveReloadRequest = (reqUrl) => + LiveServer.activeTabs.some(tab => tab.url === reqUrl && tab.isReloading); + +module.exports = LiveServer; diff --git a/packages/core/src/Site/constants.js b/packages/core/src/Site/constants.js index 10a59cbb5a..b6d9df5dd9 100644 --- a/packages/core/src/Site/constants.js +++ b/packages/core/src/Site/constants.js @@ -17,8 +17,6 @@ module.exports = { LAZY_LOADING_SITE_FILE_NAME: 'LazyLiveReloadLoadingSite.html', LAZY_LOADING_BUILD_TIME_RECOMMENDATION_LIMIT: 30000, LAZY_LOADING_REBUILD_TIME_RECOMMENDATION_LIMIT: 5000, - LAZY_LOADING_RECENTLY_VIEWED_PAGES_LIMIT: 5, - LAZY_LOADING_TIME_SINCE_LAST_REQUEST_THRESHOLD: 75, USER_VARIABLES_PATH: '_markbind/variables.md', WIKI_SITE_NAV_PATH: '_Sidebar.md', WIKI_FOOTER_PATH: '_Footer.md', diff --git a/packages/core/src/Site/index.js b/packages/core/src/Site/index.js index af7ef18fe8..5146243d1d 100644 --- a/packages/core/src/Site/index.js +++ b/packages/core/src/Site/index.js @@ -55,7 +55,6 @@ const { LAZY_LOADING_SITE_FILE_NAME, LAZY_LOADING_BUILD_TIME_RECOMMENDATION_LIMIT, LAZY_LOADING_REBUILD_TIME_RECOMMENDATION_LIMIT, - LAZY_LOADING_RECENTLY_VIEWED_PAGES_LIMIT, MARKBIND_WEBSITE_URL, MAX_CONCURRENT_PAGE_GENERATION_PROMISES, PAGE_TEMPLATE_NAME, @@ -145,7 +144,7 @@ class Site { this.currentPageViewed = onePagePath ? path.resolve(this.rootPath, FsUtil.removeExtension(onePagePath)) : ''; - this.recentlyViewedPages = onePagePath ? [this.currentPageViewed] : []; + this.currentOpenedPages = []; this.toRebuild = new Set(); } @@ -193,18 +192,10 @@ class Site { /** * Changes the site variable of the current page being viewed, building it if necessary. * @param normalizedUrl BaseUrl-less and extension-less url of the page - * @param shouldAddToRecentlyViewed Flag on whether the page should be added to recently viewed pages list * @return Boolean of whether the page needed to be rebuilt */ - changeCurrentPage(normalizedUrl, shouldAddToRecentlyViewed) { + changeCurrentPage(normalizedUrl) { this.currentPageViewed = path.join(this.rootPath, normalizedUrl); - if (shouldAddToRecentlyViewed) { - this.addToRecentlyViewedPages(this.currentPageViewed); - logger.info('Recently viewed pages, from most-to-least recent:'); - this.recentlyViewedPages.forEach((pagePath, idx) => { - logger.info(`${idx + 1}. ${utils.ensurePosix(path.relative(this.rootPath, pagePath))}`); - }); - } if (this.toRebuild.has(this.currentPageViewed)) { this.beforeSiteGenerate(); @@ -216,21 +207,21 @@ class Site { } /** - * Adds the viewed page path to the front of the recently viewed pages array, while - * also maintaining the array to keep below its specified limit. - * If the viewed page is already in the array, moves it to the front. - * @param viewedPagePath The absolute path to the page, extension-less + * Changes the list of current opened pages + * @param {Array} normalizedUrls Collection of normalized url of pages taken from the clients + * ordered from most-to-least recently opened */ - addToRecentlyViewedPages(viewedPagePath) { - const idx = this.recentlyViewedPages.indexOf(viewedPagePath); - if (idx !== -1) { - this.recentlyViewedPages.splice(idx, 1); - } - this.recentlyViewedPages.unshift(viewedPagePath); + changeCurrentOpenedPages(normalizedUrls) { + const openedPages = normalizedUrls.map(normalizedUrl => path.join(this.rootPath, normalizedUrl)); + this.currentOpenedPages = _.uniq(openedPages); - const sizeDiff = this.recentlyViewedPages.length - LAZY_LOADING_RECENTLY_VIEWED_PAGES_LIMIT; - if (sizeDiff > 0) { - this.recentlyViewedPages.splice(LAZY_LOADING_RECENTLY_VIEWED_PAGES_LIMIT, sizeDiff); + if (this.currentOpenedPages.length > 0) { + logger.info('Current opened pages, from most-to-least recent:'); + this.currentOpenedPages.forEach((pagePath, idx) => { + logger.info(`${idx + 1}. ${utils.ensurePosix(path.relative(this.rootPath, pagePath))}`); + }); + } else { + logger.info('No pages are currently opened'); } } @@ -1038,20 +1029,20 @@ class Site { } this._setTimestampVariable(); - let recentPagesToRegenerate = []; + let openedPagesToRegenerate = []; const asyncPagesToRegenerate = this.pages.filter((page) => { const doFilePathsHaveSourceFiles = filePaths.some(filePath => page.isDependency(filePath)); if (shouldRebuildAllPages || doFilePathsHaveSourceFiles) { if (this.onePagePath) { const normalizedSource = FsUtil.removeExtension(page.pageConfig.sourcePath); - const recentIdx = this.recentlyViewedPages.findIndex(pagePath => pagePath === normalizedSource); - const isRecentlyViewed = recentIdx !== -1; + const openIdx = this.currentOpenedPages.findIndex(pagePath => pagePath === normalizedSource); + const isRecentlyViewed = openIdx !== -1; if (!isRecentlyViewed) { this.toRebuild.add(normalizedSource); } else { - recentPagesToRegenerate[recentIdx] = page; + openedPagesToRegenerate[openIdx] = page; } return false; @@ -1067,9 +1058,9 @@ class Site { * As a side effect of doing assignment to an empty array, some elements might be * undefined if it has not been assigned to anything. We filter those out here. */ - recentPagesToRegenerate = recentPagesToRegenerate.filter(page => page); + openedPagesToRegenerate = openedPagesToRegenerate.filter(page => page); - const totalPagesToRegenerate = recentPagesToRegenerate.length + asyncPagesToRegenerate.length; + const totalPagesToRegenerate = openedPagesToRegenerate.length + asyncPagesToRegenerate.length; if (totalPagesToRegenerate === 0) { logger.info('No pages needed to be rebuilt'); return; @@ -1077,10 +1068,10 @@ class Site { logger.info(`Rebuilding ${totalPagesToRegenerate} pages`); const pageGenerationTasks = []; - if (recentPagesToRegenerate.length > 0) { + if (openedPagesToRegenerate.length > 0) { const recentPagesGenerationTask = { mode: 'sequential', - pages: recentPagesToRegenerate, + pages: openedPagesToRegenerate, }; pageGenerationTasks.push(recentPagesGenerationTask); } From 5751dde9ea809dcda5c7ac2bf114711e4c6d7789 Mon Sep 17 00:00:00 2001 From: Ryo Armanda Date: Sat, 20 Mar 2021 15:16:44 +0800 Subject: [PATCH 08/10] Add todo on limitation of detecting tab open or close --- packages/cli/src/lib/live-server/index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/cli/src/lib/live-server/index.js b/packages/cli/src/lib/live-server/index.js index 706e902a36..6f3a6b226a 100644 --- a/packages/cli/src/lib/live-server/index.js +++ b/packages/cli/src/lib/live-server/index.js @@ -91,6 +91,11 @@ function staticServer(root) { // CHANGED: Added line to create new entry on non-live-reload requests const reqUrl = req.originalUrl; if (!LiveServer.isLiveReloadRequest(reqUrl)) { + /* + * TODO: Find a way to handle the edge case of a tab that is immediately closed before socket + * establishment happens. Current behaviour is that the tab will remain forever in the list. + * Context: https://github.com/MarkBind/markbind/pull/1513#issuecomment-803025676 + */ const tabEntry = { url: reqUrl, client: undefined, From 7eb57c346aabe02dd316457ed2c544d1f4f88d20 Mon Sep 17 00:00:00 2001 From: Ryo Armanda Date: Sat, 20 Mar 2021 15:53:36 +0800 Subject: [PATCH 09/10] Remove sync and log opened pages on request finish --- packages/cli/index.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/cli/index.js b/packages/cli/index.js index 518de9e49e..473dce934d 100755 --- a/packages/cli/index.js +++ b/packages/cli/index.js @@ -236,15 +236,6 @@ program : urlWithoutBaseUrl; const urlWithoutExtension = fsUtil.removeExtension(urlWithIndex); - res.on('close', () => { - if (!liveServer.isLiveReloadRequest(req.originalUrl)) { - logger.info(`Opening ${fsUtil.removeExtensionPosix(req.originalUrl)}`); - const normalizedActiveUrls = liveServer.getActiveUrls() - .map(url => fsUtil.removeExtension(url)); - site.changeCurrentOpenedPages(normalizedActiveUrls); - } - }); - const didInitiateRebuild = site.changeCurrentPage(urlWithoutExtension); if (didInitiateRebuild) { req.url = utils.ensurePosix(path.join(config.baseUrl || '/', LAZY_LOADING_SITE_FILE_NAME)); From 067e5351ee7a512353310033e324edd5769e0096 Mon Sep 17 00:00:00 2001 From: Ryo Armanda Date: Sat, 20 Mar 2021 22:20:04 +0800 Subject: [PATCH 10/10] Address reviews --- packages/cli/src/lib/live-server/index.js | 4 ++-- packages/core/src/Site/index.js | 4 ++++ packages/core/src/utils/index.js | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/lib/live-server/index.js b/packages/cli/src/lib/live-server/index.js index 6f3a6b226a..5ab2237c13 100644 --- a/packages/cli/src/lib/live-server/index.js +++ b/packages/cli/src/lib/live-server/index.js @@ -88,9 +88,9 @@ function staticServer(root) { } } - // CHANGED: Added line to create new entry on non-live-reload requests + // CHANGED: Added line to create new entry on non-include and non-live-reload requests const reqUrl = req.originalUrl; - if (!LiveServer.isLiveReloadRequest(reqUrl)) { + if (!reqUrl.endsWith('._include_.html') && !LiveServer.isLiveReloadRequest(reqUrl)) { /* * TODO: Find a way to handle the edge case of a tab that is immediately closed before socket * establishment happens. Current behaviour is that the tab will remain forever in the list. diff --git a/packages/core/src/Site/index.js b/packages/core/src/Site/index.js index 5146243d1d..f667a395a3 100644 --- a/packages/core/src/Site/index.js +++ b/packages/core/src/Site/index.js @@ -212,6 +212,10 @@ class Site { * ordered from most-to-least recently opened */ changeCurrentOpenedPages(normalizedUrls) { + if (!this.onePagePath) { + return; + } + const openedPages = normalizedUrls.map(normalizedUrl => path.join(this.rootPath, normalizedUrl)); this.currentOpenedPages = _.uniq(openedPages); diff --git a/packages/core/src/utils/index.js b/packages/core/src/utils/index.js index da1a52ebdc..4246c305b9 100644 --- a/packages/core/src/utils/index.js +++ b/packages/core/src/utils/index.js @@ -120,7 +120,6 @@ module.exports = { for (let i = 0; i < array.length; i += 1) { // eslint-disable-next-line no-await-in-loop await func(array[i]); - // eslint-enable-next-line no-await-in-loop } }, };