From 6f49b1ee47cfc40bf369e317874d13916e70108c Mon Sep 17 00:00:00 2001 From: Ze Yu Date: Thu, 23 Jul 2020 19:47:54 +0800 Subject: [PATCH 1/2] Fix nunjucks path resolving Nunjucks' features that use paths resolve from the specified template directory by design. In MarkBind's case, the template directories are the respective root paths of the various sites and subsites of the project. Hence, a separate nunjucks environment with the template directory configured as such is needed for each site or subsite. Let's introduce a wrapper abstraction for this, VariableRenderer, to wrap over such a singular nunjucks environment. The VariableRenderer instances will be managed by VariableProcessor (previously VariablePreprocessor), which already has a framework set up to render variables belonging to each site or subsite. In addition, {% raw %} tags inside a layouts file do not work due to two nunjucks passes being made. Let's revert the removal of the helper functions to keep {% raw %} tags in the first pass to fix this, while searching for a solution that would allow a single pass for layouts. --- packages/core/src/Page/index.js | 21 +++-- packages/core/src/Parser.js | 12 +-- packages/core/src/Site/index.js | 6 +- packages/core/src/utils/nunjuckUtils.js | 14 --- .../VariableProcessor.js} | 50 +++++++--- .../core/src/variables/VariableRenderer.js | 93 +++++++++++++++++++ .../test_site/expected/siteData.json | 29 +++++- .../testNunjucksPathResolving.html | 80 ++++++++++++++++ .../sub_site/testNunjucksPathResolving.html | 80 ++++++++++++++++ .../test_site/expected/testLayouts.html | 2 +- .../expected/testNunjucksPathResolving.html | 80 ++++++++++++++++ test/functional/test_site/site.json | 8 ++ .../testNunjucksPathResolving.md | 23 +++++ .../testNunjucksPathResolvingInclude.md | 3 + .../sub_site/testNunjucksPathResolving.md | 23 +++++ .../test_site/testNunjucksPathResolving.md | 23 +++++ .../_markbind/layouts/default/page.md | 8 +- .../expected/index.html | 6 ++ .../test_site_expressive_layout/index.md | 13 ++- 19 files changed, 523 insertions(+), 51 deletions(-) delete mode 100644 packages/core/src/utils/nunjuckUtils.js rename packages/core/src/{preprocessors/VariablePreprocessor.js => variables/VariableProcessor.js} (89%) create mode 100644 packages/core/src/variables/VariableRenderer.js create mode 100644 test/functional/test_site/expected/sub_site/nested_sub_site/testNunjucksPathResolving.html create mode 100644 test/functional/test_site/expected/sub_site/testNunjucksPathResolving.html create mode 100644 test/functional/test_site/expected/testNunjucksPathResolving.html create mode 100644 test/functional/test_site/sub_site/nested_sub_site/testNunjucksPathResolving.md create mode 100644 test/functional/test_site/sub_site/nested_sub_site/testNunjucksPathResolvingInclude.md create mode 100644 test/functional/test_site/sub_site/testNunjucksPathResolving.md create mode 100644 test/functional/test_site/testNunjucksPathResolving.md diff --git a/packages/core/src/Page/index.js b/packages/core/src/Page/index.js index c063f54119..03663d3ad6 100644 --- a/packages/core/src/Page/index.js +++ b/packages/core/src/Page/index.js @@ -15,7 +15,6 @@ const MarkBind = require('../Parser'); const md = require('../lib/markdown-it'); const FsUtil = require('../utils/fsUtil'); -const njUtil = require('../utils/nunjuckUtils'); const utils = require('../utils'); const logger = require('../utils/logger'); @@ -573,7 +572,7 @@ class Page { } /** - * Produces expressive layouts by inserting page data into pre-specified layout + * Renders expressive layouts by inserting page data into pre-specified layout * @param pageData a page with its front matter collected * @param {FileConfig} fileConfig * @param {Parser} markbinder instance from the caller, for adding the seen sources. @@ -590,16 +589,20 @@ class Page { // Set expressive layout file as an includedFile this.includedFiles.add(layoutPagePath); return fs.readFileAsync(layoutPagePath, 'utf8') - // Include file but with altered cwf (the layout page) - // Also render MAIN_CONTENT_BODY back to itself + /* + Render {{ MAIN_CONTENT_BODY }} and {% raw/endraw %} back to itself first, + which is then dealt with in the call below to {@link renderSiteVariables}. + */ + .then(result => this.variableProcessor.renderPage(layoutPagePath, result, { + [LAYOUT_PAGE_BODY_VARIABLE]: `{{${LAYOUT_PAGE_BODY_VARIABLE}}}`, + }, true)) + // Include file with the cwf set to the layout page path .then(result => markbinder.includeFile(layoutPagePath, result, { ...fileConfig, cwf: layoutPagePath, - }, { - [LAYOUT_PAGE_BODY_VARIABLE]: `{{${LAYOUT_PAGE_BODY_VARIABLE}}}`, })) - // Insert content - .then(result => njUtil.renderRaw(result, { + // Note: The {% raw/endraw %}s previously kept are removed here. + .then(result => this.variableProcessor.renderSiteVariables(this.rootPath, result, { [LAYOUT_PAGE_BODY_VARIABLE]: pageData, })); } @@ -966,6 +969,7 @@ class Page { fixedHeader: this.fixedHeader, }; return fs.readFileAsync(this.sourcePath, 'utf-8') + .then(result => this.variableProcessor.renderPage(this.sourcePath, result)) .then(result => markbinder.includeFile(this.sourcePath, result, fileConfig)) .then((result) => { this.collectFrontMatter(result); @@ -1272,6 +1276,7 @@ class Page { cwf: file, }; return fs.readFileAsync(dependency.to, 'utf-8') + .then(result => this.variableProcessor.renderPage(dependency.to, result)) .then(result => markbinder.includeFile(dependency.to, result, fileConfig)) .then(result => Page.removeFrontMatter(result)) .then(result => this.collectPluginSources(result)) diff --git a/packages/core/src/Parser.js b/packages/core/src/Parser.js index 225df050f9..1904817697 100644 --- a/packages/core/src/Parser.js +++ b/packages/core/src/Parser.js @@ -21,7 +21,7 @@ cheerio.prototype.options.decodeEntities = false; // Don't escape HTML entities class Parser { constructor(config) { - this.variablePreprocessor = config.variablePreprocessor; + this.variableProcessor = config.variableProcessor; this.dynamicIncludeSrc = []; this.staticIncludeSrc = []; this.missingIncludeSrc = []; @@ -169,12 +169,12 @@ class Parser { } } - includeFile(file, content, config, additionalVariables = {}) { + includeFile(file, content, config) { const context = {}; context.cwf = config.cwf || file; // current working file context.callStack = []; // TODO make componentPreprocessor a class to avoid this - config.variablePreprocessor = this.variablePreprocessor; + config.variableProcessor = this.variableProcessor; return new Promise((resolve, reject) => { const handler = new htmlparser.DomHandler((error, dom) => { if (error) { @@ -196,15 +196,13 @@ class Parser { }); const parser = new htmlparser.Parser(handler); - const renderedContent = this.variablePreprocessor.renderPage(file, content, additionalVariables); - const fileExt = utils.getExt(file); if (utils.isMarkdownFileExt(fileExt)) { context.source = 'md'; - parser.parseComplete(renderedContent.toString()); + parser.parseComplete(content); } else if (fileExt === 'html') { context.source = 'html'; - parser.parseComplete(renderedContent); + parser.parseComplete(content); } else { const error = new Error(`Unsupported File Extension: '${fileExt}'`); reject(error); diff --git a/packages/core/src/Site/index.js b/packages/core/src/Site/index.js index fdb561de9e..4778a7e7ef 100644 --- a/packages/core/src/Site/index.js +++ b/packages/core/src/Site/index.js @@ -9,12 +9,12 @@ const walkSync = require('walk-sync'); const SiteConfig = require('./SiteConfig'); const Page = require('../Page'); +const VariableProcessor = require('../variables/VariableProcessor'); +const VariableRenderer = require('../variables/VariableRenderer'); const { ignoreTags } = require('../patches'); -const VariablePreprocessor = require('../preprocessors/VariablePreprocessor'); const Template = require('../../template/template'); const FsUtil = require('../utils/fsUtil'); -const njUtil = require('../utils/nunjuckUtils'); const delay = require('../utils/delay'); const logger = require('../utils/logger'); const utils = require('../utils'); @@ -125,7 +125,7 @@ class Site { // Page template path this.pageTemplatePath = path.join(__dirname, '../Page', PAGE_TEMPLATE_NAME); - this.pageTemplate = njUtil.compile(fs.readFileSync(this.pageTemplatePath, 'utf8')); + this.pageTemplate = VariableRenderer.compile(fs.readFileSync(this.pageTemplatePath, 'utf8')); this.pages = []; // Other properties diff --git a/packages/core/src/utils/nunjuckUtils.js b/packages/core/src/utils/nunjuckUtils.js deleted file mode 100644 index 9ba1a532dd..0000000000 --- a/packages/core/src/utils/nunjuckUtils.js +++ /dev/null @@ -1,14 +0,0 @@ -const nunjucks = require('nunjucks'); -const dateFilter = require('../lib/nunjucks-extensions/nunjucks-date').filter; - -const unescapedEnv = nunjucks.configure({ autoescape: false }).addFilter('date', dateFilter); - -module.exports = { - renderRaw(pageData, variableMap = {}) { - return unescapedEnv.renderString(pageData, variableMap); - }, - - compile(src, ...args) { - return nunjucks.compile(src, unescapedEnv, ...args); - }, -}; diff --git a/packages/core/src/preprocessors/VariablePreprocessor.js b/packages/core/src/variables/VariableProcessor.js similarity index 89% rename from packages/core/src/preprocessors/VariablePreprocessor.js rename to packages/core/src/variables/VariableProcessor.js index 3b5f61ff05..65f7e4e7cd 100644 --- a/packages/core/src/preprocessors/VariablePreprocessor.js +++ b/packages/core/src/variables/VariableProcessor.js @@ -9,9 +9,9 @@ _.has = require('lodash/has'); _.isArray = require('lodash/isArray'); _.isEmpty = require('lodash/isEmpty'); -const njUtil = require('../utils/nunjuckUtils'); const urlUtils = require('../utils/urls'); const logger = require('../utils/logger'); +const VariableRenderer = require('./VariableRenderer'); const { ATTRIB_CWF, @@ -43,7 +43,7 @@ const { * These methods are similar to (2), but in addition to the site variables, they * render the content with the page variables extracted from (3) as well. */ -class VariablePreprocessor { +class VariableProcessor { constructor(rootPath, baseUrlMap) { /** * Root site path @@ -63,6 +63,18 @@ class VariablePreprocessor { * @type {Object>} */ this.userDefinedVariablesMap = {}; + + /** + * Map of sites' root paths to the respective VariableRenderer instances + * @type {Object} + */ + this.variableRendererMap = {}; + + // Set up userDefinedVariablesMap and variableRendererMap + this.baseUrlMap.forEach((siteRootPath) => { + this.userDefinedVariablesMap[siteRootPath] = {}; + this.variableRendererMap[siteRootPath] = new VariableRenderer(siteRootPath); + }); } /* @@ -77,7 +89,6 @@ class VariablePreprocessor { * @param value of the variable */ addUserDefinedVariable(site, name, value) { - this.userDefinedVariablesMap[site] = this.userDefinedVariablesMap[site] || {}; this.userDefinedVariablesMap[site][name] = value; } @@ -86,7 +97,7 @@ class VariablePreprocessor { * This is to allow using previously declared site variables in site variables declared later on. */ renderAndAddUserDefinedVariable(site, name, value) { - const renderedVal = njUtil.renderRaw(value, this.userDefinedVariablesMap[site]); + const renderedVal = this.variableRendererMap[site].render(value, this.userDefinedVariablesMap[site]); this.addUserDefinedVariable(site, name, renderedVal); } @@ -100,6 +111,9 @@ class VariablePreprocessor { resetUserDefinedVariablesMap() { this.userDefinedVariablesMap = {}; + this.baseUrlMap.forEach((siteRootPath) => { + this.userDefinedVariablesMap[siteRootPath] = {}; + }); } /** @@ -124,16 +138,21 @@ class VariablePreprocessor { * @param contentFilePath of the specified content to render * @param content string to render * @param lowerPriorityVariables than the site variables, if any - * @param higherPriorityVariables than the site variables, if any + * @param higherPriorityVariables than the site variables, if any. + * Currently only used for layouts. + @param keepPercentRaw whether to reoutput {% raw/endraw %} tags, also used only for layouts. */ - renderSiteVariables(contentFilePath, content, lowerPriorityVariables = {}, higherPriorityVariables = {}) { + renderSiteVariables(contentFilePath, content, lowerPriorityVariables = {}, + higherPriorityVariables = {}, keepPercentRaw = false) { const userDefinedVariables = this.getParentSiteVariables(contentFilePath); + const parentSitePath = urlUtils.getParentSiteAbsolutePath(contentFilePath, this.rootPath, + this.baseUrlMap); - return njUtil.renderRaw(content, { + return this.variableRendererMap[parentSitePath].render(content, { ...lowerPriorityVariables, ...userDefinedVariables, ...higherPriorityVariables, - }); + }, keepPercentRaw); } /* @@ -264,7 +283,7 @@ class VariablePreprocessor { // NOTE: Selecting both at once is important to respect variable/import declaration order $('variable, import[from]').not('include > variable').each((index, elem) => { if (elem.name === 'variable') { - VariablePreprocessor.addVariable(pageVariables, elem, $(elem).html(), filePath, renderVariable); + VariableProcessor.addVariable(pageVariables, elem, $(elem).html(), filePath, renderVariable); } else { /* NOTE: we pass renderVariable here as well but not for rendering ed variables again! @@ -331,8 +350,8 @@ class VariablePreprocessor { * @param includeElement include element to extract variables from */ static extractIncludeVariables(includeElement) { - const includeInlineVariables = VariablePreprocessor.extractIncludeInlineVariables(includeElement); - const includeChildVariables = VariablePreprocessor.extractIncludeChildElementVariables(includeElement); + const includeInlineVariables = VariableProcessor.extractIncludeInlineVariables(includeElement); + const includeChildVariables = VariableProcessor.extractIncludeChildElementVariables(includeElement); return { ...includeChildVariables, @@ -358,7 +377,7 @@ class VariablePreprocessor { const fileContent = fs.readFileSync(filePath, 'utf8'); // Extract included variables from the include element, merging with the parent context variables - const includeVariables = VariablePreprocessor.extractIncludeVariables(node); + const includeVariables = VariableProcessor.extractIncludeVariables(node); // We pass in includeVariables as well to render variables used in page s // see "Test Page Variable and Included Variable Integrations" under test_site/index.md for an example @@ -396,8 +415,9 @@ class VariablePreprocessor { * @param content to render * @param highestPriorityVariables to render with the highest priority if any. * This is currently only used for the MAIN_CONTENT_BODY in layouts. + * @param keepPercentRaw whether to reoutput {% raw/endraw %} tags, also used only for layouts. */ - renderPage(contentFilePath, content, highestPriorityVariables = {}) { + renderPage(contentFilePath, content, highestPriorityVariables = {}, keepPercentRaw = false) { const { pageImportedVariables, pageVariables, @@ -406,8 +426,8 @@ class VariablePreprocessor { return this.renderSiteVariables(contentFilePath, content, { ...pageImportedVariables, ...pageVariables, - }, highestPriorityVariables); + }, highestPriorityVariables, keepPercentRaw); } } -module.exports = VariablePreprocessor; +module.exports = VariableProcessor; diff --git a/packages/core/src/variables/VariableRenderer.js b/packages/core/src/variables/VariableRenderer.js new file mode 100644 index 0000000000..381ca4e9f0 --- /dev/null +++ b/packages/core/src/variables/VariableRenderer.js @@ -0,0 +1,93 @@ +const nunjucks = require('nunjucks'); +const { filter: dateFilter } = require('../lib/nunjucks-extensions/nunjucks-date'); + +const unescapedEnv = nunjucks.configure({ autoescape: false }).addFilter('date', dateFilter); + +const START_ESCAPE_STR = '{% raw %}'; +const END_ESCAPE_STR = '{% endraw %}'; +const RAW_TAG_REGEX = new RegExp('{% *(end)?raw *%}', 'g'); + +/** + * Pads the outermost {% raw %} {% endraw %} pairs with {% raw %} {% endraw %} again. + * This is used during {@link generateExpressiveLayout}, where we do two nunjucks calls, + * one to render the layout file and another to insert the page's content. + */ +function preEscapeRawTags(pageData) { + // TODO simplify using re.matchAll once node v10 reaches 'eol' + // https://github.com/nodejs/Release#nodejs-release-working-group + const tagMatches = []; + let tagMatch = RAW_TAG_REGEX.exec(pageData); + while (tagMatch !== null) { + tagMatches.push(tagMatch); + tagMatch = RAW_TAG_REGEX.exec(pageData); + } + + const tagInfos = Array.from(tagMatches, match => ({ + isStartTag: !match[0].includes('endraw'), + index: match.index, + content: match[0], + })); + + let numStartRawTags = 0; // nesting level of {% raw %} + let lastTokenEnd = 0; + const tokens = []; + + for (let i = 0; i < tagInfos.length; i += 1) { + const { index, isStartTag, content } = tagInfos[i]; + const currentTokenEnd = index + content.length; + tokens.push(pageData.slice(lastTokenEnd, currentTokenEnd)); + lastTokenEnd = currentTokenEnd; + + if (isStartTag) { + if (numStartRawTags === 0) { + // only pad outermost {% raw %} with an extra {% raw %} + tokens.push(START_ESCAPE_STR); + } + numStartRawTags += 1; + } else { + if (numStartRawTags === 1) { + // only pad outermost {% endraw %} with an extra {% endraw %} + tokens.push(END_ESCAPE_STR); + } + numStartRawTags -= 1; + } + } + // add the last token + tokens.push(pageData.slice(lastTokenEnd)); + + return tokens.join(''); +} + +/** + * Wrapper class over a nunjucks environment configured for the respective (sub)site. + */ +class VariableRenderer { + constructor(siteRootPath) { + this.nj = nunjucks.configure(siteRootPath, { autoescape: false }).addFilter('date', dateFilter); + } + + /** + * Processes content with the instance's nunjucks environment. + * @param content to process + * @param variables to render the content with + * @param keepPercentRaw whether to keep the {% raw/endraw %} nunjucks tags + * @return {String} nunjucks processed content + */ + render(content, variables = {}, keepPercentRaw = false) { + return keepPercentRaw + ? this.nj.renderString(preEscapeRawTags(content), variables) + : this.nj.renderString(content, variables); + } + + /** + * Compiles a template specified at src independent of the template directory. + * This is used for the page template file (page.njk), where none of nunjucks' features + * involving path resolving are used. + * @param templatePath of the template to compile + */ + static compile(templatePath) { + return nunjucks.compile(templatePath, unescapedEnv); + } +} + +module.exports = VariableRenderer; diff --git a/test/functional/test_site/expected/siteData.json b/test/functional/test_site/expected/siteData.json index 7b07f459c2..e2734b1350 100644 --- a/test/functional/test_site/expected/siteData.json +++ b/test/functional/test_site/expected/siteData.json @@ -170,7 +170,7 @@ "headingKeywords": {} }, { - "title": "Hello World", + "title": "Test nunjucks path resolving", "head": "overwriteLayoutHead.md", "layout": "testLayout", "src": "testLayouts.md", @@ -179,6 +179,33 @@ "headings": {}, "headingKeywords": {} }, + { + "src": "testNunjucksPathResolving.md", + "title": "Hello World", + "layout": "default", + "globalOverrideProperty": "Overridden by global override", + "globalAndFrontMatterOverrideProperty": "Overridden by global override", + "headings": {}, + "headingKeywords": {} + }, + { + "src": "sub_site/testNunjucksPathResolving.md", + "title": "Hello World", + "layout": "default", + "globalOverrideProperty": "Overridden by global override", + "globalAndFrontMatterOverrideProperty": "Overridden by global override", + "headings": {}, + "headingKeywords": {} + }, + { + "src": "sub_site/nested_sub_site/testNunjucksPathResolving.md", + "title": "Hello World", + "layout": "default", + "globalOverrideProperty": "Overridden by global override", + "globalAndFrontMatterOverrideProperty": "Overridden by global override", + "headings": {}, + "headingKeywords": {} + }, { "src": "testAntiFOUCStyles.md", "title": "Hello World", diff --git a/test/functional/test_site/expected/sub_site/nested_sub_site/testNunjucksPathResolving.html b/test/functional/test_site/expected/sub_site/nested_sub_site/testNunjucksPathResolving.html new file mode 100644 index 0000000000..be28ad07cf --- /dev/null +++ b/test/functional/test_site/expected/sub_site/nested_sub_site/testNunjucksPathResolving.html @@ -0,0 +1,80 @@ + + + + + + + + + + Hello World + + + + + + + + + + + + + + + +
+
+
+

Test for nunjucks' various functions that use a path. + By design, nunjucks' relative paths resolve from the configured template root directory.

+

Hence, in MarkBind, these paths should also follow this behaviour. + The root directory in this case is the respective root directory of the root site or sub sites.

+
+

Test {% include %}

+

This is content from testNunjucksPathResolvingInclude.mbdf located at sub_site/nested_sub_site

+
+

Test {% import %}

+

variable to import

+
+
+ + +
+
+
+ Default footer +
+
+
+ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/functional/test_site/expected/sub_site/testNunjucksPathResolving.html b/test/functional/test_site/expected/sub_site/testNunjucksPathResolving.html new file mode 100644 index 0000000000..59997465cb --- /dev/null +++ b/test/functional/test_site/expected/sub_site/testNunjucksPathResolving.html @@ -0,0 +1,80 @@ + + + + + + + + + + Hello World + + + + + + + + + + + + + + + +
+
+
+

Test for nunjucks' various functions that use a path. + By design, nunjucks' relative paths resolve from the configured template root directory.

+

Hence, in MarkBind, these paths should also follow this behaviour. + The root directory in this case is the respective root directory of the root site or sub sites.

+
+

Test {% include %}

+

This is content from testNunjucksPathResolvingInclude.mbdf located at sub_site/nested_sub_site

+
+

Test {% import %}

+

variable to import

+
+
+ + +
+
+
+ Default footer +
+
+
+ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/functional/test_site/expected/testLayouts.html b/test/functional/test_site/expected/testLayouts.html index 10164df214..84d6825cfb 100644 --- a/test/functional/test_site/expected/testLayouts.html +++ b/test/functional/test_site/expected/testLayouts.html @@ -7,7 +7,7 @@ - Hello World + Test nunjucks path resolving diff --git a/test/functional/test_site/expected/testNunjucksPathResolving.html b/test/functional/test_site/expected/testNunjucksPathResolving.html new file mode 100644 index 0000000000..173251e028 --- /dev/null +++ b/test/functional/test_site/expected/testNunjucksPathResolving.html @@ -0,0 +1,80 @@ + + + + + + + + + + Hello World + + + + + + + + + + + + + + + +
+
+
+

Test for nunjucks' various functions that use a path. + By design, nunjucks' relative paths resolve from the configured template root directory.

+

Hence, in MarkBind, these paths should also follow this behaviour. + The root directory in this case is the respective root directory of the root site or sub sites.

+
+

Test {% include %}

+

This is content from testNunjucksPathResolvingInclude.mbdf located at sub_site/nested_sub_site

+
+

Test {% import %}

+

variable to import

+
+
+ + +
+
+
+ Default footer +
+
+
+ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/functional/test_site/site.json b/test/functional/test_site/site.json index e0c0f08afa..245798d9b4 100644 --- a/test/functional/test_site/site.json +++ b/test/functional/test_site/site.json @@ -31,6 +31,14 @@ }, { "src": "testLayouts.md", + "title": "Test nunjucks path resolving" + }, + { + "src": [ + "testNunjucksPathResolving.md", + "sub_site/testNunjucksPathResolving.md", + "sub_site/nested_sub_site/testNunjucksPathResolving.md" + ], "title": "Hello World" }, { diff --git a/test/functional/test_site/sub_site/nested_sub_site/testNunjucksPathResolving.md b/test/functional/test_site/sub_site/nested_sub_site/testNunjucksPathResolving.md new file mode 100644 index 0000000000..d050a84cad --- /dev/null +++ b/test/functional/test_site/sub_site/nested_sub_site/testNunjucksPathResolving.md @@ -0,0 +1,23 @@ +Test for nunjucks' various functions that use a path. +By design, nunjucks' relative paths resolve from the configured template root directory. + +Hence, in MarkBind, these paths should also follow this behaviour. +The root directory in this case is the respective root directory of the root site or sub sites. + +{% set includeFilePath = "testNunjucksPathResolvingInclude.md" %} + +--- + +**Test {% raw %}{% include %}{% endraw %}** + +{% include includeFilePath %} + +--- + +**Test {% raw %}{% import %}{% endraw %}** + +{% from includeFilePath import variableToImport %} + +{{ variableToImport }} + +--- diff --git a/test/functional/test_site/sub_site/nested_sub_site/testNunjucksPathResolvingInclude.md b/test/functional/test_site/sub_site/nested_sub_site/testNunjucksPathResolvingInclude.md new file mode 100644 index 0000000000..76dba64d96 --- /dev/null +++ b/test/functional/test_site/sub_site/nested_sub_site/testNunjucksPathResolvingInclude.md @@ -0,0 +1,3 @@ +This is content from testNunjucksPathResolvingInclude.mbdf located at `sub_site/nested_sub_site` + +{% set variableToImport = "variable to import" %} diff --git a/test/functional/test_site/sub_site/testNunjucksPathResolving.md b/test/functional/test_site/sub_site/testNunjucksPathResolving.md new file mode 100644 index 0000000000..b302441509 --- /dev/null +++ b/test/functional/test_site/sub_site/testNunjucksPathResolving.md @@ -0,0 +1,23 @@ +Test for nunjucks' various functions that use a path. +By design, nunjucks' relative paths resolve from the configured template root directory. + +Hence, in MarkBind, these paths should also follow this behaviour. +The root directory in this case is the respective root directory of the root site or sub sites. + +{% set includeFilePath = "nested_sub_site/testNunjucksPathResolvingInclude.md" %} + +--- + +**Test {% raw %}{% include %}{% endraw %}** + +{% include includeFilePath %} + +--- + +**Test {% raw %}{% import %}{% endraw %}** + +{% from includeFilePath import variableToImport %} + +{{ variableToImport }} + +--- diff --git a/test/functional/test_site/testNunjucksPathResolving.md b/test/functional/test_site/testNunjucksPathResolving.md new file mode 100644 index 0000000000..412ec6ce1a --- /dev/null +++ b/test/functional/test_site/testNunjucksPathResolving.md @@ -0,0 +1,23 @@ +Test for nunjucks' various functions that use a path. +By design, nunjucks' relative paths resolve from the configured template root directory. + +Hence, in MarkBind, these paths should also follow this behaviour. +The root directory in this case is the respective root directory of the root site or sub sites. + +{% set includeFilePath = "sub_site/nested_sub_site/testNunjucksPathResolvingInclude.md" %} + +--- + +**Test {% raw %}{% include %}{% endraw %}** + +{% include includeFilePath %} + +--- + +**Test {% raw %}{% import %}{% endraw %}** + +{% from includeFilePath import variableToImport %} + +{{ variableToImport }} + +--- diff --git a/test/functional/test_site_expressive_layout/_markbind/layouts/default/page.md b/test/functional/test_site_expressive_layout/_markbind/layouts/default/page.md index d724db0c83..3fe9487a50 100644 --- a/test/functional/test_site_expressive_layout/_markbind/layouts/default/page.md +++ b/test/functional/test_site_expressive_layout/_markbind/layouts/default/page.md @@ -13,4 +13,10 @@ Bottom Content - \ No newline at end of file + + + + +{% raw %} +**Test that {% raw %} and {% endraw %} tags work in the layouts file** +{% endraw %} diff --git a/test/functional/test_site_expressive_layout/expected/index.html b/test/functional/test_site_expressive_layout/expected/index.html index 7ec673d6de..e937c8d74e 100644 --- a/test/functional/test_site_expressive_layout/expected/index.html +++ b/test/functional/test_site_expressive_layout/expected/index.html @@ -35,14 +35,20 @@ Math formulas
+
+

Content above the horizontal line belongs to the layout file

Welcome to Markbind

This is a minimalistic template. To learn more about authoring contents in Markbind, visit the User Guide.

Variable from page

+

Test that {% raw %} and {% endraw %} tags work in the content file inserted into the layout

+

Content below the horizontal line belongs to the layout file

+

Bottom Content

This content can be imported to the bottom of the page

+

Test that {% raw %} and {% endraw %} tags work in the layouts file

diff --git a/test/functional/test_site_expressive_layout/index.md b/test/functional/test_site_expressive_layout/index.md index f6748e8773..092b58c429 100644 --- a/test/functional/test_site_expressive_layout/index.md +++ b/test/functional/test_site_expressive_layout/index.md @@ -4,8 +4,19 @@ Variable from page +--- +*Content above the horizontal line belongs to the layout file* + # Welcome to Markbind This is a minimalistic template. To learn more about authoring contents in Markbind, visit the [User Guide](https://markbind.org/userGuide/authoringContents.html). -{{ mainVar }} \ No newline at end of file +{{ mainVar }} + +{% raw %} +**Test that {% raw %} and {% endraw %} tags work in the content file inserted into the layout** +{% endraw %} + +*Content below the horizontal line belongs to the layout file* + +--- \ No newline at end of file From 2be282f96c71e2ee258232f1309661a3cdbd67b3 Mon Sep 17 00:00:00 2001 From: Ze Yu Date: Thu, 23 Jul 2020 19:52:10 +0800 Subject: [PATCH 2/2] Rename remaining variablePreprocessor usages --- packages/core/src/Page/index.js | 22 ++++++------- packages/core/src/Site/index.js | 20 ++++++------ .../preprocessors/componentPreprocessor.js | 4 +-- packages/core/test/unit/Parser.test.js | 32 +++++++++---------- packages/core/test/unit/Site.test.js | 10 +++--- 5 files changed, 44 insertions(+), 44 deletions(-) diff --git a/packages/core/src/Page/index.js b/packages/core/src/Page/index.js index 03663d3ad6..bb744c3dbe 100644 --- a/packages/core/src/Page/index.js +++ b/packages/core/src/Page/index.js @@ -83,7 +83,7 @@ class Page { * @property {string} pageTemplate template used for this page * @property {string} title * @property {string} titlePrefix https://markbind.org/userGuide/siteConfiguration.html#titleprefix - * @property {VariablePreprocessor} variablePreprocessor + * @property {VariableProcessor} variableProcessor * @property {string} sourcePath the source file for rendering this page * @property {string} tempPath the temp path for writing intermediate result * @property {string} resultPath the output path of this page @@ -177,9 +177,9 @@ class Page { */ this.titlePrefix = pageConfig.titlePrefix; /** - * @type {VariablePreprocessor} + * @type {VariableProcessor} */ - this.variablePreprocessor = pageConfig.variablePreprocessor; + this.variableProcessor = pageConfig.variableProcessor; /** * The source file for rendering this page @@ -641,7 +641,7 @@ class Page { // Set header file as an includedFile this.includedFiles.add(headerPath); - const renderedHeader = this.variablePreprocessor.renderSiteVariables(this.sourcePath, headerContent); + const renderedHeader = this.variableProcessor.renderSiteVariables(this.sourcePath, headerContent); return `${renderedHeader}\n${pageData}`; } @@ -670,7 +670,7 @@ class Page { // Set footer file as an includedFile this.includedFiles.add(footerPath); - const renderedFooter = this.variablePreprocessor.renderSiteVariables(this.sourcePath, footerContent); + const renderedFooter = this.variableProcessor.renderSiteVariables(this.sourcePath, footerContent); return `${pageData}\n${renderedFooter}`; } @@ -702,7 +702,7 @@ class Page { } this.includedFiles.add(siteNavPath); - const siteNavMappedData = this.variablePreprocessor.renderSiteVariables(this.sourcePath, siteNavContent); + const siteNavMappedData = this.variableProcessor.renderSiteVariables(this.sourcePath, siteNavContent); // Check navigation elements const $ = cheerio.load(siteNavMappedData); @@ -893,8 +893,8 @@ class Page { // Set head file as an includedFile this.includedFiles.add(headFilePath); - const headFileMappedData = this.variablePreprocessor.renderSiteVariables(this.sourcePath, - headFileContent).trim(); + const headFileMappedData = this.variableProcessor.renderSiteVariables(this.sourcePath, + headFileContent).trim(); // Split top and bottom contents const $ = cheerio.load(headFileMappedData); if ($('head-top').length) { @@ -947,7 +947,7 @@ class Page { * @typedef {Object} FileConfig * @property {Set} baseUrlMap the set of urls representing the sites' base directories * @property {string} rootPath - * @property {VariablePreprocessor} variablePreprocessor + * @property {VariableProcessor} variableProcessor * @property {Object} headerIdMap * @property {boolean} fixedHeader indicates whether the header of the page is fixed */ @@ -956,7 +956,7 @@ class Page { this.includedFiles = new Set([this.sourcePath]); this.headerIdMap = {}; // Reset for live reload const markbinder = new MarkBind({ - variablePreprocessor: this.variablePreprocessor, + variableProcessor: this.variableProcessor, }); /** * @type {FileConfig} @@ -1263,7 +1263,7 @@ class Page { * so that we only recursively rebuild the file's included content */ const markbinder = new MarkBind({ - variablePreprocessor: this.variablePreprocessor, + variableProcessor: this.variableProcessor, }); /** * @type {FileConfig} diff --git a/packages/core/src/Site/index.js b/packages/core/src/Site/index.js index 4778a7e7ef..9b2d858d6f 100644 --- a/packages/core/src/Site/index.js +++ b/packages/core/src/Site/index.js @@ -139,8 +139,8 @@ class Site { this.siteConfig = undefined; this.siteConfigPath = siteConfigPath; - // Site wide variable preprocessor - this.variablePreprocessor = undefined; + // Site wide variable processor + this.variableProcessor = undefined; // Lazy reload properties this.onePagePath = onePagePath; @@ -278,7 +278,7 @@ class Site { title: config.title || '', titlePrefix: this.siteConfig.titlePrefix, headingIndexingLevel: this.siteConfig.headingIndexingLevel, - variablePreprocessor: this.variablePreprocessor, + variableProcessor: this.variableProcessor, sourcePath, resultPath, asset: { @@ -520,7 +520,7 @@ class Site { .map(x => path.resolve(this.rootPath, x)); this.baseUrlMap = new Set(candidates.map(candidate => path.dirname(candidate))); - this.variablePreprocessor = new VariablePreprocessor(this.rootPath, this.baseUrlMap); + this.variableProcessor = new VariableProcessor(this.rootPath, this.baseUrlMap); return Promise.resolve(); } @@ -529,7 +529,7 @@ class Site { * Collects the user defined variables map in the site/subsites */ collectUserDefinedVariablesMap() { - this.variablePreprocessor.resetUserDefinedVariablesMap(); + this.variableProcessor.resetUserDefinedVariablesMap(); this.baseUrlMap.forEach((base) => { const userDefinedVariablesPath = path.resolve(base, USER_VARIABLES_PATH); @@ -550,8 +550,8 @@ class Site { const siteBaseUrl = siteRelativePathFromRoot === '' ? this.siteConfig.baseUrl : path.posix.join(this.siteConfig.baseUrl || '/', siteRelativePathFromRoot); - this.variablePreprocessor.addUserDefinedVariable(base, 'baseUrl', siteBaseUrl); - this.variablePreprocessor.addUserDefinedVariable(base, 'MarkBind', MARKBIND_LINK_HTML); + this.variableProcessor.addUserDefinedVariable(base, 'baseUrl', siteBaseUrl); + this.variableProcessor.addUserDefinedVariable(base, 'MarkBind', MARKBIND_LINK_HTML); const $ = cheerio.load(content, { decodeEntities: false }); $('variable,span').each((index, element) => { @@ -564,13 +564,13 @@ class Site { const jsonData = fs.readFileSync(variableFilePath); const varData = JSON.parse(jsonData); Object.entries(varData).forEach(([varName, varValue]) => { - this.variablePreprocessor.renderAndAddUserDefinedVariable(base, varName, varValue); + this.variableProcessor.renderAndAddUserDefinedVariable(base, varName, varValue); }); } catch (err) { logger.warn(`Error ${err.message}`); } } else { - this.variablePreprocessor.renderAndAddUserDefinedVariable(base, name, $(element).html()); + this.variableProcessor.renderAndAddUserDefinedVariable(base, name, $(element).html()); } }); }); @@ -1311,7 +1311,7 @@ class Site { timeZoneName: 'short', }; const time = new Date().toLocaleTimeString(this.siteConfig.locale, options); - this.variablePreprocessor.addUserDefinedVariableForAllSites('timestamp', time); + this.variableProcessor.addUserDefinedVariableForAllSites('timestamp', time); return Promise.resolve(); } } diff --git a/packages/core/src/preprocessors/componentPreprocessor.js b/packages/core/src/preprocessors/componentPreprocessor.js index afa4db4626..24d8470eef 100644 --- a/packages/core/src/preprocessors/componentPreprocessor.js +++ b/packages/core/src/preprocessors/componentPreprocessor.js @@ -235,11 +235,11 @@ function _preprocessInclude(node, context, config, parser) { const isIncludeSrcMd = _isHtmlIncludingMarkdown(element, context, filePath); - const { variablePreprocessor } = config; + const { variableProcessor } = config; const { renderedContent, childContext, - } = variablePreprocessor.renderIncludeFile(actualFilePath, element, context, filePath); + } = variableProcessor.renderIncludeFile(actualFilePath, element, context, filePath); _deleteIncludeAttributes(element); diff --git a/packages/core/test/unit/Parser.test.js b/packages/core/test/unit/Parser.test.js index 9422559db1..fdb8836ea6 100644 --- a/packages/core/test/unit/Parser.test.js +++ b/packages/core/test/unit/Parser.test.js @@ -1,18 +1,18 @@ const path = require('path'); const fs = require('fs'); const Parser = require('../../src/Parser'); -const VariablePreprocessor = require('../../src/preprocessors/VariablePreprocessor'); +const VariableProcessor = require('../../src/variables/VariableProcessor'); jest.mock('fs'); afterEach(() => fs.vol.reset()); const ROOT_PATH = path.resolve(''); -function getNewDefaultVariablePreprocessor() { - const DEFAULT_VARIABLE_PREPROCESSOR = new VariablePreprocessor(ROOT_PATH, new Set([ROOT_PATH])); - DEFAULT_VARIABLE_PREPROCESSOR.addUserDefinedVariable(ROOT_PATH, 'baseUrl', '{{baseUrl}}'); +function getNewDefaultVariableProcessor() { + const DEFAULT_VARIABLE_PROCESSOR = new VariableProcessor(ROOT_PATH, new Set([ROOT_PATH])); + DEFAULT_VARIABLE_PROCESSOR.addUserDefinedVariable(ROOT_PATH, 'baseUrl', '{{baseUrl}}'); - return DEFAULT_VARIABLE_PREPROCESSOR; + return DEFAULT_VARIABLE_PROCESSOR; } test('includeFile replaces with
', async () => { @@ -35,7 +35,7 @@ test('includeFile replaces with
', async () => { fs.vol.fromJSON(json, ''); const baseUrlMap = new Set([ROOT_PATH]); - const markbinder = new Parser({ variablePreprocessor: getNewDefaultVariablePreprocessor() }); + const markbinder = new Parser({ variableProcessor: getNewDefaultVariableProcessor() }); const result = await markbinder.includeFile(indexPath, index, { baseUrlMap, rootPath: ROOT_PATH, @@ -74,7 +74,7 @@ test('includeFile replaces with
', async fs.vol.fromJSON(json, ''); const baseUrlMap = new Set([ROOT_PATH]); - const markbinder = new Parser({ variablePreprocessor: getNewDefaultVariablePreprocessor() }); + const markbinder = new Parser({ variableProcessor: getNewDefaultVariableProcessor() }); const result = await markbinder.includeFile(indexPath, index, { baseUrlMap, rootPath: ROOT_PATH, @@ -109,7 +109,7 @@ test('includeFile replaces with empty < fs.vol.fromJSON(json, ''); const baseUrlMap = new Set([ROOT_PATH]); - const markbinder = new Parser({ variablePreprocessor: getNewDefaultVariablePreprocessor() }); + const markbinder = new Parser({ variableProcessor: getNewDefaultVariableProcessor() }); const result = await markbinder.includeFile(indexPath, index, { baseUrlMap, rootPath: ROOT_PATH, @@ -147,7 +147,7 @@ test('includeFile replaces with
', async fs.vol.fromJSON(json, ''); const baseUrlMap = new Set([ROOT_PATH]); - const markbinder = new Parser({ variablePreprocessor: getNewDefaultVariablePreprocessor() }); + const markbinder = new Parser({ variableProcessor: getNewDefaultVariableProcessor() }); const result = await markbinder.includeFile(indexPath, index, { baseUrlMap, rootPath: ROOT_PATH, @@ -189,7 +189,7 @@ test('includeFile replaces with inline fs.vol.fromJSON(json, ''); const baseUrlMap = new Set([ROOT_PATH]); - const markbinder = new Parser({ variablePreprocessor: getNewDefaultVariablePreprocessor() }); + const markbinder = new Parser({ variableProcessor: getNewDefaultVariableProcessor() }); const result = await markbinder.includeFile(indexPath, index, { baseUrlMap, rootPath: ROOT_PATH, @@ -228,7 +228,7 @@ test('includeFile replaces with trimmed c fs.vol.fromJSON(json, ''); const baseUrlMap = new Set([ROOT_PATH]); - const markbinder = new Parser({ variablePreprocessor: getNewDefaultVariablePreprocessor() }); + const markbinder = new Parser({ variableProcessor: getNewDefaultVariableProcessor() }); const result = await markbinder.includeFile(indexPath, index, { baseUrlMap, rootPath: ROOT_PATH, @@ -270,7 +270,7 @@ test('includeFile replaces with error with
fs.vol.fromJSON(json, ''); const baseUrlMap = new Set([ROOT_PATH]); - const markbinder = new Parser({ variablePreprocessor: getNewDefaultVariablePreprocessor() }); + const markbinder = new Parser({ variableProcessor: getNewDefaultVariableProcessor() }); const result = await markbinder.includeFile(indexPath, index, { baseUrlMap, rootPath: ROOT_PATH, @@ -347,7 +347,7 @@ test('includeFile replaces with fs.vol.fromJSON(json, ''); const baseUrlMap = new Set([ROOT_PATH]); - const markbinder = new Parser({ variablePreprocessor: getNewDefaultVariablePreprocessor() }); + const markbinder = new Parser({ variableProcessor: getNewDefaultVariableProcessor() }); const result = await markbinder.includeFile(indexPath, index, { baseUrlMap, rootPath: ROOT_PATH, @@ -400,7 +400,7 @@ test('includeFile detects cyclic references for static cyclic includes', async ( `\t${indexPath}`, ].join('\n'); - const markbinder = new Parser({ variablePreprocessor: getNewDefaultVariablePreprocessor() }); + const markbinder = new Parser({ variableProcessor: getNewDefaultVariableProcessor() }); const result = await markbinder.includeFile(indexPath, index, { baseUrlMap, rootPath: ROOT_PATH, @@ -412,7 +412,7 @@ test('includeFile detects cyclic references for static cyclic includes', async ( }); test('renderFile converts markdown headers to

', async () => { - const markbinder = new Parser({ variablePreprocessor: getNewDefaultVariablePreprocessor() }); + const markbinder = new Parser({ variableProcessor: getNewDefaultVariableProcessor() }); const rootPath = path.resolve(''); const headerIdMap = {}; const indexPath = path.resolve('index.md'); diff --git a/packages/core/test/unit/Site.test.js b/packages/core/test/unit/Site.test.js index 1b2b2873cc..1b7692e1f3 100644 --- a/packages/core/test/unit/Site.test.js +++ b/packages/core/test/unit/Site.test.js @@ -326,7 +326,7 @@ test('Site resolves variables referencing other variables', async () => { await site.collectBaseUrl(); await site.collectUserDefinedVariablesMap(); - const root = site.variablePreprocessor.userDefinedVariablesMap[path.resolve('')]; + const root = site.variableProcessor.userDefinedVariablesMap[path.resolve('')]; // check all variables expect(root.level1).toEqual('variable'); @@ -357,10 +357,10 @@ test('Site read correct user defined variables', async () => { await site.collectBaseUrl(); await site.collectUserDefinedVariablesMap(); - const root = site.variablePreprocessor.userDefinedVariablesMap[path.resolve('')]; - const sub = site.variablePreprocessor.userDefinedVariablesMap[path.resolve('sub')]; - const subsub = site.variablePreprocessor.userDefinedVariablesMap[path.resolve('sub/sub')]; - const othersub = site.variablePreprocessor.userDefinedVariablesMap[path.resolve('otherSub/sub')]; + const root = site.variableProcessor.userDefinedVariablesMap[path.resolve('')]; + const sub = site.variableProcessor.userDefinedVariablesMap[path.resolve('sub')]; + const subsub = site.variableProcessor.userDefinedVariablesMap[path.resolve('sub/sub')]; + const othersub = site.variableProcessor.userDefinedVariablesMap[path.resolve('otherSub/sub')]; // check all baseUrls expect(root.baseUrl).toEqual('');