From 64390dc0d3744f610b845fbdd0b686ec823329db Mon Sep 17 00:00:00 2001 From: tinyAdapter Date: Sat, 19 Oct 2024 16:59:14 +0800 Subject: [PATCH 1/2] feat: rewrite preprocessor & add force-multiline --- packages/parser/test/parser.test.ts | 66 ++++----- packages/parser/test/parserMultiline.test.ts | 148 +++++++++++++++++++ 2 files changed, 181 insertions(+), 33 deletions(-) create mode 100644 packages/parser/test/parserMultiline.test.ts diff --git a/packages/parser/test/parser.test.ts b/packages/parser/test/parser.test.ts index 5df62f23f..05161f6b4 100644 --- a/packages/parser/test/parser.test.ts +++ b/packages/parser/test/parser.test.ts @@ -1,9 +1,9 @@ import SceneParser from "../src/index"; -import {ADD_NEXT_ARG_LIST, SCRIPT_CONFIG} from "../src/config/scriptConfig"; -import {expect, test} from "vitest"; -import {commandType, ISentence} from "../src/interface/sceneInterface"; -import * as fsp from 'fs/promises' -import {fileType} from "../src/interface/assets"; +import { ADD_NEXT_ARG_LIST, SCRIPT_CONFIG } from "../src/config/scriptConfig"; +import { expect, test } from "vitest"; +import { commandType, ISentence } from "../src/interface/sceneInterface"; +import * as fsp from 'fs/promises'; +import { fileType } from "../src/interface/assets"; test("label", async () => { @@ -21,7 +21,7 @@ test("label", async () => { commandRaw: "label", content: "end", args: [ - {key: "next", value: true} + { key: "next", value: true } ], sentenceAssets: [], subScene: [] @@ -45,10 +45,10 @@ test("args", async () => { commandRaw: "changeFigure", content: "m2.png", args: [ - {key: "left", value: true}, - {key: "next", value: true} + { key: "left", value: true }, + { key: "next", value: true } ], - sentenceAssets: [{name: "m2.png", url: 'm2.png', type: fileType.figure, lineNumber: 0}], + sentenceAssets: [{ name: "m2.png", url: 'm2.png', type: fileType.figure, lineNumber: 0 }], subScene: [] }; expect(result.sentenceList).toContainEqual(expectSentenceItem); @@ -86,16 +86,16 @@ test("long-script", async () => { return fileName; }, ADD_NEXT_ARG_LIST, SCRIPT_CONFIG); - console.log('line count:', sceneText.split('\n').length) - console.time('parse-time-consumed') + console.log('line count:', sceneText.split('\n').length); + console.time('parse-time-consumed'); const result = parser.parse(sceneText, "start", "/start.txt"); - console.timeEnd('parse-time-consumed') + console.timeEnd('parse-time-consumed'); const expectSentenceItem: ISentence = { command: commandType.label, commandRaw: "label", content: "end", args: [ - {key: "next", value: true} + { key: "next", value: true } ], sentenceAssets: [], subScene: [] @@ -118,7 +118,7 @@ test("var", async () => { command: commandType.say, commandRaw: "WebGAL", content: "a=1?", - args: [{key: 'speaker', value: 'WebGAL'}, {key: 'when', value: "a==1"}], + args: [{ key: 'speaker', value: 'WebGAL' }, { key: 'when', value: "a==1" }], sentenceAssets: [], subScene: [] }; @@ -137,18 +137,18 @@ Game_key:0f86dstRf; Title_img:WebGAL_New_Enter_Image.png; Title_bgm:s_Title.mp3; Title_logos: 1.png | 2.png | Image Logo.png| -show -active=false -add=op! -count=3;This is a fake config, do not reference anything. - `) + `); expect(configFesult).toContainEqual({ command: 'Title_logos', args: ['1.png', '2.png', 'Image Logo.png'], options: [ - {key: 'show', value: true}, - {key: 'active', value: false}, - {key: 'add', value: 'op!'}, - {key: 'count', value: 3}, + { key: 'show', value: true }, + { key: 'active', value: false }, + { key: 'add', value: 'op!' }, + { key: 'count', value: 3 }, ] - }) -}) + }); +}); test("config-stringify", async () => { const parser = new SceneParser((assetList) => { @@ -162,20 +162,20 @@ Game_key:0f86dstRf; Title_img:WebGAL_New_Enter_Image.png; Title_bgm:s_Title.mp3; Title_logos: 1.png | 2.png | Image Logo.png| -show -active=false -add=op! -count=3;This is a fake config, do not reference anything. - `) + `); const stringifyResult = parser.stringifyConfig(configFesult); - const configResult2 = parser.parseConfig(stringifyResult) + const configResult2 = parser.parseConfig(stringifyResult); expect(configResult2).toContainEqual({ command: 'Title_logos', args: ['1.png', '2.png', 'Image Logo.png'], options: [ - {key: 'show', value: true}, - {key: 'active', value: false}, - {key: 'add', value: 'op!'}, - {key: 'count', value: 3}, + { key: 'show', value: true }, + { key: 'active', value: false }, + { key: 'add', value: 'op!' }, + { key: 'count', value: 3 }, ] - }) -}) + }); +}); test("say statement", async () => { @@ -184,13 +184,13 @@ test("say statement", async () => { return fileName; }, ADD_NEXT_ARG_LIST, SCRIPT_CONFIG); - const result = parser.parse(`say:123 -speaker=xx;`,'test','test') + const result = parser.parse(`say:123 -speaker=xx;`, 'test', 'test'); expect(result.sentenceList).toContainEqual({ command: commandType.say, commandRaw: "say", content: "123", - args: [{key: 'speaker', value: 'xx'}], + args: [{ key: 'speaker', value: 'xx' }], sentenceAssets: [], subScene: [] - }) -}) + }); +}); diff --git a/packages/parser/test/parserMultiline.test.ts b/packages/parser/test/parserMultiline.test.ts new file mode 100644 index 000000000..42815bcd2 --- /dev/null +++ b/packages/parser/test/parserMultiline.test.ts @@ -0,0 +1,148 @@ +import { sceneTextPreProcess } from "../src/sceneTextPreProcessor"; +import { expect, test } from "vitest"; + +test("parser-multiline-basic", async () => { + const testScene = `changeFigure:a.png -left + -next + -id=id1 + +saySomething`; + const expected = `changeFigure:a.png -left -next -id=id1 +;_WEBGAL_LINE_BREAK_ -next +;_WEBGAL_LINE_BREAK_ -id=id1 + +saySomething`; + + const preprocessedScene = sceneTextPreProcess(testScene); + expect(preprocessedScene).toEqual(expected); +}); + + +test("parser-multiline-disable-when-encounter-concat-1", async () => { + const testScene = `intro:aaa + |bbb -concat +`; + const expected = `intro:aaa + |bbb -concat +`; + + const preprocessedScene = sceneTextPreProcess(testScene); + expect(preprocessedScene).toEqual(expected); +}); + + +test("parser-multiline-disable-when-encounter-concat-2", async () => { + const testScene = `intro:aaa + |bbb + |ccc -concat +`; + const expected = `intro:aaa|bbb +;_WEBGAL_LINE_BREAK_ |bbb + |ccc -concat +`; + + const preprocessedScene = sceneTextPreProcess(testScene); + expect(preprocessedScene).toEqual(expected); +}); + +test("parser-multiline-user-force-allow-multiline-in-concat", async () => { + const testScene = String.raw`intro:aaa\ +|bbb\ +|ccc -concat +`; + const expected = `intro:aaa|bbb|ccc -concat +;_WEBGAL_LINE_BREAK_|bbb +;_WEBGAL_LINE_BREAK_|ccc -concat +`; + + const preprocessedScene = sceneTextPreProcess(testScene); + expect(preprocessedScene).toEqual(expected); +}); + +test("parser-multiline-others-same-as-before", async () => { + const testScene = `听起来是不是非常吸引人？ -v4.wav; +changeFigure:none -right -next; +setAnimation:l2c -target=fig-left -next; +WebGAL 引擎也具有动画系统和特效系统，使用 WebGAL 开发的游戏可以拥有很好的表现效果。 -v5.wav; +`; + + const preprocessedScene = sceneTextPreProcess(testScene); + expect(preprocessedScene).toEqual(testScene); +}); + +test("parser-multiline-full", async () => { + const testScene = `changeFigure:a.png -left + -next + -id=id1 + +intro:aaa + |bbb|ccc + |ddd + -next; + +; WebGAL 引擎会默认读取 start.txt 作为初始场景，因此请不要删除，并在初始场景内跳转到其他场景 +bgm:s_Title.mp3; +unlockBgm:s_Title.mp3 -name=雲を追いかけて; +intro:你好 +|欢迎来到 WebGAL 的世界; +changeBg:bg.png -next; +unlockCg:bg.png -name=良夜; // 解锁CG并赋予名称 +changeFigure:stand.png -left -next; +setAnimation:enter-from-left + -target=fig-left -next; +WebGAL:欢迎使用 WebGAL！这是一款全新的网页端视觉小说引擎。 + -v1.wav; +changeFigure:stand2.png + -right -next; +WebGAL 是使用 Web 技术开发的引擎，因此在网页端有良好的表现。 -v2.wav; +由于这个特性，如果你将 WebGAL 部署到服务器或网页托管平台上，玩家只需要一串链接就可以开始游玩！ -v3.wav; +setAnimation:move-front-and-back + -target=fig-left + -next; + +听起来是不是非常吸引人？ -v4.wav; +changeFigure:none -right -next; +setAnimation:l2c -target=fig-left -next; +WebGAL 引擎也具有动画系统和特效系统，使用 WebGAL 开发的游戏可以拥有很好的表现效果。 + -v5.wav; +`; + + const expected = `changeFigure:a.png -left -next -id=id1 +;_WEBGAL_LINE_BREAK_ -next +;_WEBGAL_LINE_BREAK_ -id=id1 + +intro:aaa|bbb|ccc|ddd -next; +;_WEBGAL_LINE_BREAK_ |bbb|ccc +;_WEBGAL_LINE_BREAK_ |ddd +;_WEBGAL_LINE_BREAK_ -next; + +; WebGAL 引擎会默认读取 start.txt 作为初始场景，因此请不要删除，并在初始场景内跳转到其他场景 +bgm:s_Title.mp3; +unlockBgm:s_Title.mp3 -name=雲を追いかけて; +intro:你好 +|欢迎来到 WebGAL 的世界; +changeBg:bg.png -next; +unlockCg:bg.png -name=良夜; // 解锁CG并赋予名称 +changeFigure:stand.png -left -next; +setAnimation:enter-from-left -target=fig-left -next; +;_WEBGAL_LINE_BREAK_ -target=fig-left -next; +WebGAL:欢迎使用 WebGAL！这是一款全新的网页端视觉小说引擎。 -v1.wav; +;_WEBGAL_LINE_BREAK_ -v1.wav; +changeFigure:stand2.png -right -next; +;_WEBGAL_LINE_BREAK_ -right -next; +WebGAL 是使用 Web 技术开发的引擎，因此在网页端有良好的表现。 -v2.wav; +由于这个特性，如果你将 WebGAL 部署到服务器或网页托管平台上，玩家只需要一串链接就可以开始游玩！ -v3.wav; +setAnimation:move-front-and-back -target=fig-left -next; +;_WEBGAL_LINE_BREAK_ -target=fig-left +;_WEBGAL_LINE_BREAK_ -next; + +听起来是不是非常吸引人？ -v4.wav; +changeFigure:none -right -next; +setAnimation:l2c -target=fig-left -next; +WebGAL 引擎也具有动画系统和特效系统，使用 WebGAL 开发的游戏可以拥有很好的表现效果。 -v5.wav; +;_WEBGAL_LINE_BREAK_ -v5.wav; +`; + + const preprocessedScene = sceneTextPreProcess(testScene); + expect(preprocessedScene).toEqual(expected); +}); From 1b3f3f9d4ac313c2a0b70d3debdc4183bc525468 Mon Sep 17 00:00:00 2001 From: tinyAdapter Date: Sat, 19 Oct 2024 16:59:37 +0800 Subject: [PATCH 2/2] test: parser: multiline --- packages/parser/src/index.ts | 4 +- packages/parser/src/sceneTextPreProcessor.ts | 267 +++++++++++++++---- 2 files changed, 213 insertions(+), 58 deletions(-) diff --git a/packages/parser/src/index.ts b/packages/parser/src/index.ts index 3121f10cf..030e1bc7b 100644 --- a/packages/parser/src/index.ts +++ b/packages/parser/src/index.ts @@ -9,7 +9,7 @@ import { fileType } from './interface/assets'; import { IAsset } from './interface/sceneInterface'; import { sceneParser } from './sceneParser'; import { IWebGALStyleObj, scss2cssinjsParser } from "./styleParser"; -import { sceneTextPreProcess } from "@/sceneTextPreProcessor"; +import { sceneTextPreProcess } from "./sceneTextPreProcessor"; export default class SceneParser { private readonly SCRIPT_CONFIG_MAP: ConfigMap; @@ -77,4 +77,4 @@ export default class SceneParser { } export { ADD_NEXT_ARG_LIST, SCRIPT_CONFIG }; -export { sceneTextPreProcess } +export { sceneTextPreProcess }; diff --git a/packages/parser/src/sceneTextPreProcessor.ts b/packages/parser/src/sceneTextPreProcessor.ts index 7a0bf2792..244c2c91b 100644 --- a/packages/parser/src/sceneTextPreProcessor.ts +++ b/packages/parser/src/sceneTextPreProcessor.ts @@ -1,68 +1,223 @@ +/** + * Preprocessor for scene text. + * + * Use two-pass to generate a new scene text that concats multiline sequences + * into a single line and add placeholder lines to preserve the original number + * of lines. + * + * @param sceneText The original scene text + * @returns The processed scene text + */ export function sceneTextPreProcess(sceneText: string): string { - const lines = sceneText.replaceAll('\r', '').split('\n'); + let lines = sceneText.replaceAll('\r', '').split('\n'); + + lines = sceneTextPreProcessPassOne(lines); + lines = sceneTextPreProcessPassTwo(lines); + + return lines.join('\n'); +} + +/** + * Pass one. + * + * Add escape character to all lines that should be multiline. + * + * @param lines The original lines + * @returns The processed lines + */ +function sceneTextPreProcessPassOne(lines: string[]): string[] { + const processedLines: string[] = []; + let lastLineIsMultiline = false; + let thisLineIsMultiline = false; + + for (const line of lines) { + thisLineIsMultiline = false; + + if (canBeMultiline(line)) { + thisLineIsMultiline = true; + } + + if (shouldNotBeMultiline(line, lastLineIsMultiline)) { + thisLineIsMultiline = false; + } + + if (thisLineIsMultiline) { + processedLines[processedLines.length - 1] += '\\'; + } + + processedLines.push(line); + + lastLineIsMultiline = thisLineIsMultiline; + } + + return processedLines; +} + +function canBeMultiline(line: string): boolean { + if (!line.startsWith(' ')) { + return false; + } + + const trimmedLine = line.trimStart(); + return trimmedLine.startsWith('|') || trimmedLine.startsWith('-'); +} + +/** + * Logic to check if a line should not be multiline. + * + * @param line The line to check + * @returns If the line should not be multiline + */ +function shouldNotBeMultiline(line: string, lastLineIsMultiline: boolean): boolean { + if (!lastLineIsMultiline && isEmptyLine(line)) { + return true; + } + + // Custom logic: if the line contains -concat, it should not be multiline + if (line.indexOf('-concat') !== -1) { + return true; + } + + return false; +} + +function isEmptyLine(line: string): boolean { + return line.trim() === ''; +} + + +/** + * Pass two. + * + * Traverse the lines to + * - remove escape characters + * - add placeholder lines to preserve the original number of lines. + * + * @param lines The lines in pass one + * @returns The processed lines + */ +function sceneTextPreProcessPassTwo(lines: string[]): string[] { const processedLines: string[] = []; - let lastNonMultilineIndex = -1; - let isInMultilineSequence = false; + let currentMultilineContent = ""; + let placeHolderLines: string[] = []; - function isMultiline(line: string): boolean { - if (!line.startsWith(' ')) return false - const trimmedLine = line.trimStart(); - return trimmedLine.startsWith('|') || trimmedLine.startsWith('-'); + function concat(line: string) { + let trimmed = line.trim(); + if (trimmed.startsWith('-')) { + trimmed = " " + trimmed; + } + currentMultilineContent = currentMultilineContent + trimmed; + placeHolderLines.push(placeholderLine(line)); } - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; - - if (line.trim() === '') { - // Empty line handling - if (isInMultilineSequence) { - // Check if the next line is a multiline line - - let isStillInMulti = false; - for (let j = i + 1; j < lines.length; j++) { - const lookForwardLine = lines[j] || '' - // 遇到正常语句了，直接中断 - if (lookForwardLine.trim() !== '' && !isMultiline(lookForwardLine)) { - isStillInMulti = false; - break; - } - // 必须找到后面接的是参数，并且中间没有遇到任何正常语句才行 - if (lookForwardLine.trim() !== '' && isMultiline(lookForwardLine)) { - isStillInMulti = true; - break; - } - } - if (isStillInMulti) { - // Still within a multiline sequence - processedLines.push(';_WEBGAL_LINE_BREAK_'); - } else { - // End of multiline sequence - isInMultilineSequence = false; - processedLines.push(line); - } + for (const line of lines) { + console.log(line); + if (line.endsWith('\\')) { + const trueLine = line.slice(0, -1); + + if (currentMultilineContent === "") { + // first line + currentMultilineContent = trueLine; } else { - // Preserve empty lines outside of multiline sequences - processedLines.push(line); - } - } else if (isMultiline(line)) { - // Multiline statement handling - if (lastNonMultilineIndex >= 0) { - // Concatenate to the previous non-multiline statement - const trimedLine = line.trimStart() - const addBlank = trimedLine.startsWith('-') ? ' ' : ''; - processedLines[lastNonMultilineIndex] += addBlank + trimedLine; + // middle line + concat(trueLine); } + continue; + } - // Add the special comment line - processedLines.push(';_WEBGAL_LINE_BREAK_' + line); - isInMultilineSequence = true; - } else { - // Non-multiline statement handling - processedLines.push(line); - lastNonMultilineIndex = processedLines.length - 1; - isInMultilineSequence = false; + if (currentMultilineContent !== "") { + // end line + concat(line); + processedLines.push(currentMultilineContent); + processedLines.push(...placeHolderLines); + + placeHolderLines = []; + currentMultilineContent = ""; + continue; } + + processedLines.push(line); } - return processedLines.join('\n'); + return processedLines; +} + +/** + * Placeholder Line. Adding this line preserves the original number of lines + * in the scene text, so that it can be compatible with the graphical editor. + * + * @param content The original content on this line + * @returns The placeholder line + */ +function placeholderLine(content = "") { + return ";_WEBGAL_LINE_BREAK_" + content; } + +// export function sceneTextPreProcess(sceneText: string): string { +// const lines = sceneText.replaceAll('\r', '').split('\n'); +// const processedLines: string[] = []; +// let lastNonMultilineIndex = -1; +// let isInMultilineSequence = false; + +// function isMultiline(line: string): boolean { +// if (!line.startsWith(' ')) return false; +// const trimmedLine = line.trimStart(); +// return trimmedLine.startsWith('|') || trimmedLine.startsWith('-'); +// } + +// for (let i = 0; i < lines.length; i++) { +// const line = lines[i]; + +// if (line.trim() === '') { +// // Empty line handling +// if (isInMultilineSequence) { +// // Check if the next line is a multiline line + +// let isStillInMulti = false; +// for (let j = i + 1; j < lines.length; j++) { +// const lookForwardLine = lines[j] || ''; +// // 遇到正常语句了，直接中断 +// if (lookForwardLine.trim() !== '' && !isMultiline(lookForwardLine)) { +// isStillInMulti = false; +// break; +// } +// // 必须找到后面接的是参数，并且中间没有遇到任何正常语句才行 +// if (lookForwardLine.trim() !== '' && isMultiline(lookForwardLine)) { +// isStillInMulti = true; +// break; +// } +// } +// if (isStillInMulti) { +// // Still within a multiline sequence +// processedLines.push(';_WEBGAL_LINE_BREAK_'); +// } else { +// // End of multiline sequence +// isInMultilineSequence = false; +// processedLines.push(line); +// } +// } else { +// // Preserve empty lines outside of multiline sequences +// processedLines.push(line); +// } +// } else if (isMultiline(line)) { +// // Multiline statement handling +// if (lastNonMultilineIndex >= 0) { +// // Concatenate to the previous non-multiline statement +// const trimedLine = line.trimStart(); +// const addBlank = trimedLine.startsWith('-') ? ' ' : ''; +// processedLines[lastNonMultilineIndex] += addBlank + trimedLine; +// } + +// // Add the special comment line +// processedLines.push(';_WEBGAL_LINE_BREAK_' + line); +// isInMultilineSequence = true; +// } else { +// // Non-multiline statement handling +// processedLines.push(line); +// lastNonMultilineIndex = processedLines.length - 1; +// isInMultilineSequence = false; +// } +// } + +// return processedLines.join('\n'); +// } \ No newline at end of file