From 467f9c652fa4db1be7c02508e3f2143f21eb0de6 Mon Sep 17 00:00:00 2001 From: ComfyUI Wiki Date: Sun, 5 Jul 2026 23:21:37 +0800 Subject: [PATCH 1/2] Add i18n hash sync and code-fence repair tooling. Manual translation edits can refresh hashes without re-running translate, auto-chunk applies to more pages, and unclosed ``` blocks can be fixed structurally before falling back to full re-translation. --- .cursor/skills/docs-i18n-translate/SKILL.md | 24 +- .github/scripts/i18n/README.md | 42 +- .../i18n/check-translation-truncation.ts | 54 ++- .github/scripts/i18n/repair-fences-i18n.ts | 133 ++++++ .github/scripts/i18n/repair-fences.test.ts | 47 ++ .github/scripts/i18n/sync-hash-i18n.ts | 435 ++++++++++++++++++ .github/scripts/i18n/translation-config.json | 4 +- package.json | 2 + 8 files changed, 734 insertions(+), 7 deletions(-) create mode 100644 .github/scripts/i18n/repair-fences-i18n.ts create mode 100644 .github/scripts/i18n/repair-fences.test.ts create mode 100644 .github/scripts/i18n/sync-hash-i18n.ts diff --git a/.cursor/skills/docs-i18n-translate/SKILL.md b/.cursor/skills/docs-i18n-translate/SKILL.md index 747dc37f5..c7d337da6 100644 --- a/.cursor/skills/docs-i18n-translate/SKILL.md +++ b/.cursor/skills/docs-i18n-translate/SKILL.md @@ -52,7 +52,9 @@ Requires **Bun**. | `pnpm translate:snippets` | Snippets only | | `pnpm translate -- --pages-only` | Skip snippets | | `pnpm translate:check-truncation` | Scan for truncated output | +| `pnpm translate:repair-fences` | Append missing closing ``` (no API) | | `pnpm translate:repair-truncated -- --lang ko` | Re-translate flagged files | +| `pnpm translate:sync-hash` | Refresh hashes after manual zh/ja/ko edits (no API) | | `pnpm translate:sync-docs-json` | Sync `docs.json` nav paths (labels preserved) | | `pnpm translate:sync-docs-json -- --translate-nav-labels` | Also translate new EN nav labels | | `pnpm glossary:sync` | Rebuild glossary from ComfyUI frontend | @@ -70,6 +72,20 @@ pnpm translate -- changelog/index.mdx # or specific paths pnpm translate:check-truncation # if long page / changelog ``` +### Small English edits (manual translation) + +When only a line or paragraph changed: + +```bash +# 1. Edit English + update zh/ja/ko by hand (or ask Cursor to patch matching sections) +# 2. Sync hashes so translate skips the file +pnpm translate:sync-hash -- path/to/page.mdx +pnpm translate:sync-hash -- --verify path/to/page.mdx # optional sanity check +``` + +For larger or new sections, use `pnpm translate -- path/to/page.mdx` (chunked pages +only re-translate changed `##` sections when `auto_chunk` applies). + ### Changelog (`changelog/index.mdx`) - Strategy: `update_blocks` (configured in `translation-config.json`) @@ -96,7 +112,7 @@ pnpm translate -- changelog/index.mdx --lang zh | Strategy | When | Config | |----------|------|--------| -| `heading_sections` | Long reference pages | `chunked_files` or `auto_chunk` (≥10k chars, ≥4 `##`) | +| `heading_sections` | Long reference pages | `chunked_files` or `auto_chunk` (≥3k chars, ≥2 `##`) | | `update_blocks` | Changelog | `chunked_files` entry for `changelog/index.mdx` | Checkpoints per block — safe to resume after interrupt. @@ -134,7 +150,8 @@ pnpm glossary:sync # after frontend locale updates When user updates English docs and needs translations: - [ ] Identify changed files (or run `pnpm translate:dry-run`) -- [ ] Run `pnpm translate` for affected paths — **not** `cms:prepare` unless CMS/Strapi +- [ ] For small edits: hand-update translations, then `pnpm translate:sync-hash -- ` +- [ ] For larger edits: run `pnpm translate` for affected paths — **not** `cms:prepare` unless CMS/Strapi - [ ] For changelog, translate **docs** `zh/changelog/` etc., not CMS staging - [ ] After long pages, run `pnpm translate:check-truncation` - [ ] Commit translated MDX + updated `translationSourceHash` / `translationBlockHashes` @@ -148,6 +165,7 @@ When user updates English docs and needs translations: |------|------| | `.github/scripts/i18n/translate-i18n.ts` | Entry point | | `.github/scripts/i18n/chunked-translate.ts` | Block splitting/reassembly | +| `.github/scripts/i18n/sync-hash-i18n.ts` | Hash-only sync after manual edits | | `.github/scripts/i18n/translation-config.json` | Languages, skip paths, chunked files | | `.github/scripts/i18n/glossary.mjs` | Term injection | | `.github/scripts/i18n/README.md` | Full reference | @@ -158,7 +176,9 @@ When user updates English docs and needs translations: | Issue | Fix | |-------|-----| | File skipped | English hash unchanged — use `pnpm translate:force` or edit EN source | +| Manual translation done | `pnpm translate:sync-hash -- ` to refresh hashes | | Truncated translation | `translate:repair-truncated` or add to `chunked_files` | +| Missing closing ``` only | `translate:repair-fences` (structural); re-translate if code inside block was cut | | Wrong term | `glossary/overrides/{lang}.json` or `preserve_terms` | | PR i18n comment | Run `pnpm translate` for listed files | | Changelog date still English | Re-run translate for that block; dates derived from EN | diff --git a/.github/scripts/i18n/README.md b/.github/scripts/i18n/README.md index a960be843..35d27f423 100644 --- a/.github/scripts/i18n/README.md +++ b/.github/scripts/i18n/README.md @@ -41,7 +41,9 @@ pnpm translate -- --lang zh,ja # specific languages pnpm translate -- installation/x.mdx # specific files pnpm translate:snippets # snippets only pnpm translate:check-truncation # scan for truncated output +pnpm translate:repair-fences # append missing closing ``` pnpm translate:repair-truncated -- --lang ko +pnpm translate:sync-hash # update hashes after manual translation edits pnpm translate:sync-docs-json # sync docs.json navigation paths ``` @@ -49,6 +51,21 @@ Quality controls during/after a run write to `.github/i18n-logs/translate/` (gitignored): semantic mismatches reported by the model, and a truncation scan (`check-translation-truncation.ts`). +Truncation heuristics include **unclosed ` ``` ` blocks** (`unclosed_code_fence`), +fewer fence markers than English (`missing_code_fence`), and short body length. + +```bash +pnpm translate:check-truncation # scan all languages +pnpm translate:check-truncation -- --lang ko +pnpm translate:repair-fences # append missing closing ``` +pnpm translate:repair-fences -- --dry-run # preview fence fixes +pnpm translate:repair-truncated -- --lang ko # re-translate via API when content was cut +``` + +`repair-fences` is a **structural** fix (adds `\`\`\`` at the end). It does not +restore code lines lost inside the block. Use `repair-truncated` when the block +body itself was truncated. + ### Long pages (chunked translation) Very long MDX files (e.g. `tutorials/partner-nodes/pricing.mdx`) exceed model @@ -60,7 +77,7 @@ output limits when translated in one shot. Two strategies avoid truncation: | `update_blocks` | Changelog | `` blocks | Per-block content hash in `translationBlockHashes` (new labels + EN edits) | Configure explicit paths in `translation-config.json` → `chunked_files`, or rely -on `auto_chunk` (default: body ≥ 10k chars and ≥ 4 `##` sections) to auto-enable +on `auto_chunk` (default: body ≥ 3k chars and ≥ 2 `##` sections) to auto-enable `heading_sections`. Changelog `` dates are **derived from English** and @@ -87,6 +104,28 @@ pnpm translate:check-truncation -- --lang ko pnpm translate:repair-truncated -- --lang ko # force re-translate flagged files ``` +### Sync hashes after manual edits + +When you edit English and update zh/ja/ko translations by hand (or with Cursor), refresh +metadata so `pnpm translate` skips those files: + +```bash +pnpm translate:sync-hash -- path/to/page.mdx # specific file(s) +pnpm translate:sync-hash -- --lang zh path/to/page.mdx +pnpm translate:sync-hash -- --dry-run path/to/page.mdx # preview +pnpm translate:sync-hash -- --verify path/to/page.mdx # warn if EN blocks still look pending +pnpm translate:sync-hash # all files with hash drift +``` + +This updates `translationSourceHash` (and `translationBlockHashes` on chunked pages) +from the English source. It does **not** call the translation API or change prose. + +Example: + +```bash +pnpm translate:sync-hash -- installation/system_requirements.mdx +``` + ## Quality review (LLM-as-a-judge) `review-i18n.ts` scores existing translations with an **independent** (and @@ -198,6 +237,7 @@ env → `frontend_locales_path` in `translation-config.json` → |------|------| | `translate-i18n.ts` | translation entry point | | `chunked-translate.ts` | split/reassemble long MDX (`heading_sections`, `update_blocks`) | +| `sync-hash-i18n.ts` | Refresh translation hashes after manual edits (no API) | | `sync-glossary.mjs` | rebuild the glossary frontend mirror | | `glossary.mjs` | load glossary layers, select + inject terms | | `i18n-config.mjs` | shared path rules from `translation-config.json` | diff --git a/.github/scripts/i18n/check-translation-truncation.ts b/.github/scripts/i18n/check-translation-truncation.ts index daa52e92d..a8fddc680 100644 --- a/.github/scripts/i18n/check-translation-truncation.ts +++ b/.github/scripts/i18n/check-translation-truncation.ts @@ -105,6 +105,53 @@ function enFenceCount(enBody: string): number { return countFencePairs(enBody); } +/** True when body has an odd number of ``` line markers (unclosed block). */ +export function hasUnclosedCodeFence(body: string): boolean { + return countCodeFences(body).open; +} + +/** Append a closing ``` when the body ends inside a code block. */ +export function repairUnclosedCodeFence(body: string): { + body: string; + repaired: boolean; + openLang: string; + openLine: number; +} { + const fence = countCodeFences(body); + if (!fence.open) { + return { body, repaired: false, openLang: "", openLine: 0 }; + } + const trimmed = body.endsWith("\n") ? body : `${body}\n`; + return { + body: `${trimmed}\`\`\`\n`, + repaired: true, + openLang: fence.openLang, + openLine: fence.openLine, + }; +} + +/** Fix an MDX file whose body ends with an unclosed ``` block. Frontmatter is preserved. */ +export function repairTargetContent(targetContent: string): { + content: string; + repaired: boolean; + detail: string; +} { + const { frontmatter, body } = parseFrontmatterAndBody(targetContent); + const result = repairUnclosedCodeFence(body); + if (!result.repaired) { + return { content: targetContent, repaired: false, detail: "" }; + } + const newBody = result.body; + const raw = frontmatter ? `${frontmatter}\n${newBody}` : newBody; + const content = raw.endsWith("\n") ? raw : `${raw}\n`; + const langHint = result.openLang ? `\`${result.openLang}\` ` : ""; + return { + content, + repaired: true, + detail: `appended closing \`\`\` for ${langHint}block opened near line ${result.openLine}`, + }; +} + function targetPath(enRel: string, lang: LangConfig): string { if (enRel.startsWith("snippets/")) { return join(ROOT, lang.snippets_dir, enRel.slice("snippets/".length)); @@ -354,7 +401,8 @@ export async function writeTruncationReport( "changelog missing blocks.", "", "Repair:", - " npm run translate:repair-truncated -- --lang ko", + " npm run translate:repair-fences # append missing closing ```", + " npm run translate:repair-truncated -- --lang ko # re-translate truncated files", "", `Note: semantic AI review notes (mismatch) are separate — see ${TRANSLATE_LOG_REL}/mismatches.txt`, " (only written when `npm run translate` runs and the model reports issues).", @@ -416,7 +464,9 @@ async function main() { console.log(`\nLog: ${TXT_PATH}`); console.log(`JSON: ${JSON_PATH}`); if (issues.length > 0) { - console.log("\nRepair: npm run translate:repair-truncated -- --lang "); + console.log("\nRepair:"); + console.log(" npm run translate:repair-fences"); + console.log(" npm run translate:repair-truncated -- --lang "); } } diff --git a/.github/scripts/i18n/repair-fences-i18n.ts b/.github/scripts/i18n/repair-fences-i18n.ts new file mode 100644 index 000000000..56fc858b7 --- /dev/null +++ b/.github/scripts/i18n/repair-fences-i18n.ts @@ -0,0 +1,133 @@ +#!/usr/bin/env bun +/** + * Append missing closing ``` markers in translated MDX (structural fix only). + * + * Safe to run when `translate:check-truncation` reports `unclosed_code_fence`. + * Does not restore truncated code inside the block — use `translate:repair-truncated` + * when content was cut off, not just the closing fence. + * + * Usage: + * pnpm translate:repair-fences + * pnpm translate:repair-fences -- --lang ko + * pnpm translate:repair-fences -- --dry-run path/to/page.mdx + */ + +import { readFile, writeFile } from "fs/promises"; +import { join } from "path"; +import { + loadI18nConfig, + REPO_ROOT, + parseLangArg as parseLangArgFromConfig, +} from "./i18n-config.mjs"; +import { + hasUnclosedCodeFence, + repairTargetContent, + scanTruncationIssues, + writeTruncationReport, +} from "./check-translation-truncation.ts"; +import { parseFrontmatterAndBody } from "./chunked-translate.ts"; + +interface LangConfig { + code: string; + name: string; + dir: string; + snippets_dir: string; +} + +const config = loadI18nConfig() as { + languages: LangConfig[]; + skip_paths: string[]; +}; + +async function readFileOr(path: string, fallback = ""): Promise { + try { + return await readFile(path, "utf-8"); + } catch { + return fallback; + } +} + +function parseLangArg(args: string[]): LangConfig[] { + try { + return parseLangArgFromConfig(args, config.languages) as LangConfig[]; + } catch (err: unknown) { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); + } +} + +async function main() { + const args = process.argv.slice(2); + const dryRun = args.includes("--dry-run"); + const snippetsMode = args.includes("--snippets"); + const langs = parseLangArg(args); + const fileArgs = args.filter((a, i) => !a.startsWith("--") && args[i - 1] !== "--lang"); + + const issues = await scanTruncationIssues({ langs, snippetsMode, fileArgs }); + const fenceIssues = issues.filter((i) => i.reasons.includes("unclosed_code_fence")); + + if (fenceIssues.length === 0) { + console.log("No unclosed code fences found."); + return; + } + + console.log(`Found ${fenceIssues.length} file(s) with unclosed \`\`\` blocks${dryRun ? " [dry-run]" : ""}:`); + + let repaired = 0; + let skipped = 0; + const repairedPairs: { lang: string; enRel: string }[] = []; + + for (const issue of fenceIssues) { + const targetPath = join(REPO_ROOT, issue.targetRel); + const targetContent = await readFileOr(targetPath); + if (!targetContent.trim()) { + skipped++; + continue; + } + + const { body } = parseFrontmatterAndBody(targetContent); + if (!hasUnclosedCodeFence(body)) { + console.log(` [${issue.lang}] ${issue.enRel}: already balanced, skipping`); + skipped++; + continue; + } + + const result = repairTargetContent(targetContent); + if (!result.repaired) { + skipped++; + continue; + } + + if (dryRun) { + console.log(` [${issue.lang}] would repair: ${issue.targetRel}`); + console.log(` ${result.detail}`); + } else { + await writeFile(targetPath, result.content); + console.log(` [${issue.lang}] repaired: ${issue.targetRel}`); + console.log(` ${result.detail}`); + repaired++; + repairedPairs.push({ lang: issue.lang, enRel: issue.enRel }); + } + } + + if (!dryRun && repairedPairs.length > 0) { + const remaining = await scanTruncationIssues({ + langs, + snippetsMode, + pairs: repairedPairs.map((p) => ({ langCode: p.lang, enRel: p.enRel })), + }); + await writeTruncationReport(remaining, { replacePairs: repairedPairs }); + } + + console.log( + `\nDone: ${dryRun ? fenceIssues.length : repaired} repaired, ${skipped} skipped` + + (fenceIssues.some((i) => i.reasons.includes("missing_code_fence")) && !dryRun + ? "\nNote: files with missing_code_fence (fewer fences than EN) may need translate:repair-truncated." + : "") + ); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/.github/scripts/i18n/repair-fences.test.ts b/.github/scripts/i18n/repair-fences.test.ts new file mode 100644 index 000000000..0fb303bcd --- /dev/null +++ b/.github/scripts/i18n/repair-fences.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, test } from "bun:test"; +import { + hasUnclosedCodeFence, + repairTargetContent, + repairUnclosedCodeFence, +} from "./check-translation-truncation.ts"; + +describe("repairUnclosedCodeFence", () => { + test("detects unclosed fence", () => { + const body = "Some text\n\n```bash\necho hello\n"; + expect(hasUnclosedCodeFence(body)).toBe(true); + }); + + test("leaves balanced fences unchanged", () => { + const body = "Some text\n\n```bash\necho hello\n```\n"; + expect(hasUnclosedCodeFence(body)).toBe(false); + const result = repairUnclosedCodeFence(body); + expect(result.repaired).toBe(false); + }); + + test("appends closing fence", () => { + const body = "Some text\n\n```bash\necho hello\n"; + const result = repairUnclosedCodeFence(body); + expect(result.repaired).toBe(true); + expect(result.body.endsWith("```\n")).toBe(true); + expect(hasUnclosedCodeFence(result.body)).toBe(false); + }); +}); + +describe("repairTargetContent", () => { + test("preserves frontmatter and appends closing fence", () => { + const input = `--- +title: "Example" +--- + +Intro + +\`\`\`python +print("hi") +`; + const { content, repaired, detail } = repairTargetContent(input); + expect(repaired).toBe(true); + expect(content.startsWith("---\ntitle: \"Example\"")).toBe(true); + expect(content.trimEnd().endsWith("```")).toBe(true); + expect(detail).toContain("appended closing"); + }); +}); diff --git a/.github/scripts/i18n/sync-hash-i18n.ts b/.github/scripts/i18n/sync-hash-i18n.ts new file mode 100644 index 000000000..a45432a1a --- /dev/null +++ b/.github/scripts/i18n/sync-hash-i18n.ts @@ -0,0 +1,435 @@ +#!/usr/bin/env bun +/** + * Sync translationSourceHash / translationBlockHashes from English source without + * calling the translation API. Use after manually editing zh/ja/ko translations. + * + * Usage: + * pnpm translate:sync-hash # all pending (hash drift) + * pnpm translate:sync-hash -- installation/foo.mdx # specific file(s) + * pnpm translate:sync-hash -- --lang zh path/to/page.mdx + * pnpm translate:sync-hash -- --dry-run # preview only + * pnpm translate:sync-hash -- --verify path/to/page.mdx # warn if EN changed blocks look untranslated + */ + +import { readdir, readFile, writeFile, mkdir } from "fs/promises"; +import { createHash } from "crypto"; +import { join, dirname, relative } from "path"; +import { + loadI18nConfig, + REPO_ROOT, + stripLangPrefix, + isEnglishPagePath, + isEnglishSnippetPath, + parseLangArg as parseLangArgFromConfig, +} from "./i18n-config.mjs"; +import { + type AutoChunkConfig, + type ChunkedFileConfig, + type ChunkStrategy, + aggregateDocumentHash, + canonicalBlockLabelOrder, + changelogLabelHash, + documentBlockHashes, + getSectionSyncStatus, + parseDocument, + parseFrontmatterAndBody, + resolveChunkStrategy, + setChunkedTranslationMeta, + stripTranslationMetaFromFrontmatter, +} from "./chunked-translate.ts"; + +const config = loadI18nConfig() as TranslationConfig; +const pathFilterOpts = { languages: config.languages, skip_paths: config.skip_paths ?? [] }; +const CHUNKED_FILES: ChunkedFileConfig[] = config.chunked_files ?? []; +const AUTO_CHUNK: AutoChunkConfig | undefined = config.auto_chunk; + +interface TranslationConfig { + languages: LangConfig[]; + skip_paths?: string[]; + chunked_files?: ChunkedFileConfig[]; + auto_chunk?: AutoChunkConfig; +} + +interface LangConfig { + code: string; + name: string; + dir: string; + snippets_dir: string; +} + +function sourceHash(en: string): string { + return createHash("sha256").update(en).digest("hex").slice(0, 8); +} + +function getExistingHash(content: string): string | null { + const fmMatch = content.match(/translationSourceHash:\s*"?([a-f0-9]{8})"?/); + if (fmMatch) return fmMatch[1] ?? null; + const mdxComment = content.match(/\{\/\*\s*translationSourceHash:\s*([a-f0-9]{8})\s*\*\/\}/); + if (mdxComment) return mdxComment[1] ?? null; + const htmlComment = content.match(//); + return htmlComment?.[1] ?? null; +} + +function setTranslationMeta(content: string, hash: string, enPath: string): string { + const metaBlock = [`translationSourceHash: ${hash}`, `translationFrom: ${enPath}`].join("\n"); + const fmMatch = content.match(/^(---\n)([\s\S]*?)(\n---)/); + if (!fmMatch) { + return `---\n${metaBlock}\n---\n${content}`; + } + const [, open, body, close] = fmMatch; + const rest = content.slice(fmMatch[0].length); + const cleaned = stripTranslationMetaFromFrontmatter(body); + return `${open}${cleaned}\n${metaBlock}${close}${rest}`; +} + +function setSnippetHash(content: string, hash: string): string { + const stripped = content + .replace(/\{\/\*\s*translationSourceHash:\s*[a-f0-9]{8}\s*\*\/\}\n?/, "") + .replace(/\n?/, ""); + return `{/* translationSourceHash: ${hash} */}\n${stripped}`; +} + +async function readFileOr(path: string, fallback = ""): Promise { + try { + return await readFile(path, "utf-8"); + } catch { + return fallback; + } +} + +async function collectMdx(dir: string): Promise { + const results: string[] = []; + const entries = await readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + results.push(...(await collectMdx(full))); + } else if (entry.name.endsWith(".mdx")) { + results.push(full); + } + } + return results; +} + +function parseLangArg(args: string[]): LangConfig[] { + try { + return parseLangArgFromConfig(args, config.languages) as LangConfig[]; + } catch (err: unknown) { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); + } +} + +function resolveFileChunkStrategy(relPath: string, enContent: string): ChunkStrategy | null { + const { body } = parseFrontmatterAndBody(enContent); + return resolveChunkStrategy(relPath, body, CHUNKED_FILES, AUTO_CHUNK); +} + +interface PathMapping { + enPath: string; + targetPath: string; + enRel: string; + targetRel: string; +} + +function makeMapping(lang: LangConfig, relPath: string, snippetsMode: boolean): PathMapping { + const enRel = stripLangPrefix(relPath, config.languages); + if (snippetsMode) { + return { + enPath: join(REPO_ROOT, "snippets", enRel), + targetPath: join(REPO_ROOT, lang.snippets_dir, enRel), + enRel: `snippets/${enRel}`, + targetRel: `${lang.snippets_dir}/${enRel}`, + }; + } + return { + enPath: join(REPO_ROOT, enRel), + targetPath: join(REPO_ROOT, lang.dir, enRel), + enRel, + targetRel: `${lang.dir}/${enRel}`, + }; +} + +function syncChunkedHashes( + enContent: string, + targetContent: string, + strategy: ChunkStrategy, + enRel: string +): string { + const enDoc = parseDocument(enContent, strategy); + const enBlockHashes = documentBlockHashes(enDoc.blocks); + const fileHash = + strategy === "update_blocks" + ? changelogLabelHash(enDoc.blocks) + : aggregateDocumentHash(enBlockHashes); + + const { frontmatter, body } = parseFrontmatterAndBody(targetContent); + const labelOrder = canonicalBlockLabelOrder( + strategy, + enDoc.blocks.map((b) => b.label) + ); + const bodyText = body.endsWith("\n") ? body : `${body}\n`; + const raw = `${frontmatter}\n${bodyText}`; + return setChunkedTranslationMeta(raw, fileHash, enRel, enBlockHashes, labelOrder); +} + +function syncPlainHashes( + targetContent: string, + enContent: string, + enRel: string, + snippetsMode: boolean +): string { + const hash = sourceHash(enContent); + if (snippetsMode) { + return setSnippetHash(targetContent, hash); + } + return setTranslationMeta(targetContent, hash, enRel); +} + +interface SyncResult { + status: "updated" | "skipped" | "missing" | "unchanged"; + warnings: string[]; +} + +function computeSyncedContent( + enContent: string, + targetContent: string, + relPath: string, + enRel: string, + snippetsMode: boolean +): { output: string; expectedFileHash: string } { + const chunkStrategy = !snippetsMode ? resolveFileChunkStrategy(relPath, enContent) : null; + if (chunkStrategy) { + const enDoc = parseDocument(enContent, chunkStrategy); + const enBlockHashes = documentBlockHashes(enDoc.blocks); + const expectedFileHash = + chunkStrategy === "update_blocks" + ? changelogLabelHash(enDoc.blocks) + : aggregateDocumentHash(enBlockHashes); + return { + output: syncChunkedHashes(enContent, targetContent, chunkStrategy, enRel), + expectedFileHash, + }; + } + return { + output: syncPlainHashes(targetContent, enContent, enRel, snippetsMode), + expectedFileHash: sourceHash(enContent), + }; +} + +function collectVerifyWarnings( + enContent: string, + targetContent: string, + relPath: string, + langCode: string, + snippetsMode: boolean +): string[] { + const warnings: string[] = []; + const chunkStrategy = !snippetsMode ? resolveFileChunkStrategy(relPath, enContent) : null; + + if (chunkStrategy) { + const status = getSectionSyncStatus(enContent, targetContent, chunkStrategy, false, langCode); + if (status.pendingBlocks.length > 0) { + const unit = chunkStrategy === "update_blocks" ? "block(s)" : "section(s)"; + warnings.push( + `English changed in ${status.pendingBlocks.length} ${unit}: ${status.pendingBlocks.slice(0, 5).join(", ")}${status.pendingBlocks.length > 5 ? ", …" : ""}. Confirm translations were updated manually.` + ); + } + return warnings; + } + + const expected = sourceHash(enContent); + const existing = getExistingHash(targetContent); + if (existing && existing !== expected) { + warnings.push( + "English source hash differs from stored translationSourceHash. Confirm the full translation was updated manually." + ); + } + return warnings; +} + +async function syncOneFile( + relPath: string, + lang: LangConfig, + snippetsMode: boolean, + dryRun: boolean, + verify: boolean +): Promise { + const { enPath, targetPath, enRel, targetRel } = makeMapping(lang, relPath, snippetsMode); + const enContent = await readFileOr(enPath); + if (!enContent || enContent.length < 50) { + return { status: "skipped", warnings: [] }; + } + + const targetContent = await readFileOr(targetPath); + if (!targetContent.trim()) { + console.log(` [${lang.code}] missing target: ${targetRel}`); + return { status: "missing", warnings: [] }; + } + + const { output, expectedFileHash } = computeSyncedContent( + enContent, + targetContent, + relPath, + enRel, + snippetsMode + ); + + const existingHash = getExistingHash(targetContent); + if (existingHash === expectedFileHash && output === targetContent) { + return { status: "unchanged", warnings: [] }; + } + + const warnings = verify + ? collectVerifyWarnings(enContent, targetContent, relPath, lang.code, snippetsMode) + : []; + + if (dryRun) { + console.log(` [${lang.code}] would sync hash: ${targetRel}`); + for (const w of warnings) console.log(` ⚠ ${w}`); + return { status: "updated", warnings }; + } + + await mkdir(dirname(targetPath), { recursive: true }); + await writeFile(targetPath, output); + console.log(` [${lang.code}] synced hash: ${targetRel}`); + for (const w of warnings) console.log(` ⚠ ${w}`); + return { status: "updated", warnings }; +} + +async function collectEnglishFiles(snippetsMode: boolean, fileArgs: string[]): Promise { + if (fileArgs.length > 0) { + return fileArgs + .map((f) => stripLangPrefix(f, config.languages)) + .filter((f) => + snippetsMode + ? isEnglishSnippetPath(f.replace(/^snippets\//, ""), pathFilterOpts) + : isEnglishPagePath(f, pathFilterOpts) + ); + } + + if (snippetsMode) { + const all = await collectMdx(join(REPO_ROOT, "snippets")); + return all + .map((f) => relative(join(REPO_ROOT, "snippets"), f)) + .filter((f) => isEnglishSnippetPath(f, pathFilterOpts)); + } + + const all = await collectMdx(REPO_ROOT); + return all + .map((f) => relative(REPO_ROOT, f)) + .filter((f) => isEnglishPagePath(f, pathFilterOpts)); +} + +function needsHashSync( + enContent: string, + targetContent: string, + relPath: string, + snippetsMode: boolean +): boolean { + if (!targetContent.trim()) return false; + const chunkStrategy = !snippetsMode ? resolveFileChunkStrategy(relPath, enContent) : null; + if (chunkStrategy) { + const status = getSectionSyncStatus(enContent, targetContent, chunkStrategy, false); + return !status.upToDate; + } + return getExistingHash(targetContent) !== sourceHash(enContent); +} + +async function main() { + const args = process.argv.slice(2); + const dryRun = args.includes("--dry-run"); + const verify = args.includes("--verify"); + const allFiles = args.includes("--all"); + const snippetsOnly = args.includes("--snippets") || args.includes("--snippets-only"); + const pagesOnly = args.includes("--pages-only") || args.includes("--no-snippets"); + const selectedLangs = parseLangArg(args); + const fileArgs = args.filter((a, i) => !a.startsWith("--") && args[i - 1] !== "--lang"); + + const phases: { snippetsMode: boolean; fileArgs: string[]; label: string }[] = []; + if (snippetsOnly) { + phases.push({ snippetsMode: true, fileArgs, label: "snippets" }); + } else if (pagesOnly) { + phases.push({ snippetsMode: false, fileArgs, label: "pages" }); + } else if (fileArgs.length > 0) { + const pageArgs = fileArgs.filter((f) => !f.replace(/^snippets\//, "").startsWith("snippets/")); + const snippetArgs = fileArgs.filter((f) => f.startsWith("snippets/")); + if (pageArgs.length > 0) phases.push({ snippetsMode: false, fileArgs: pageArgs, label: "pages" }); + if (snippetArgs.length > 0) phases.push({ snippetsMode: true, fileArgs: snippetArgs, label: "snippets" }); + if (phases.length === 0) phases.push({ snippetsMode: false, fileArgs, label: "pages" }); + } else { + phases.push({ snippetsMode: false, fileArgs: [], label: "pages" }); + phases.push({ snippetsMode: true, fileArgs: [], label: "snippets" }); + } + + let totalUpdated = 0; + let totalUnchanged = 0; + let totalMissing = 0; + let totalWarnings = 0; + + for (const phase of phases) { + console.log(`\n=== sync-hash (${phase.label})${dryRun ? " [dry-run]" : ""} ===`); + const files = await collectEnglishFiles(phase.snippetsMode, phase.fileArgs); + if (files.length === 0) { + console.log(`No ${phase.label} files to process.`); + continue; + } + + for (const relPath of files) { + let fileHasDrift = false; + for (const lang of selectedLangs) { + const { enPath, targetPath } = makeMapping(lang, relPath, phase.snippetsMode); + const enContent = await readFileOr(enPath); + const targetContent = await readFileOr(targetPath); + if (!enContent || enContent.length < 50) continue; + if (!allFiles && fileArgs.length === 0 && !needsHashSync(enContent, targetContent, relPath, phase.snippetsMode)) { + continue; + } + fileHasDrift = true; + } + + if (!fileHasDrift && fileArgs.length === 0 && !allFiles) continue; + + if (fileHasDrift || fileArgs.length > 0 || allFiles) { + console.log(`${relPath}${phase.snippetsMode ? " (snippet)" : ""}:`); + } + + for (const lang of selectedLangs) { + const { enPath, targetPath } = makeMapping(lang, relPath, phase.snippetsMode); + const enContent = await readFileOr(enPath); + const targetContent = await readFileOr(targetPath); + if (!enContent || enContent.length < 50) continue; + if (!allFiles && fileArgs.length === 0 && !needsHashSync(enContent, targetContent, relPath, phase.snippetsMode)) { + totalUnchanged++; + continue; + } + + const result = await syncOneFile(relPath, lang, phase.snippetsMode, dryRun, verify); + if (result.status === "updated") totalUpdated++; + else if (result.status === "unchanged") { + if (fileArgs.length > 0 || allFiles) { + const { targetRel } = makeMapping(lang, relPath, phase.snippetsMode); + console.log(` [${lang.code}] already in sync: ${targetRel}`); + } + totalUnchanged++; + } + else if (result.status === "missing") totalMissing++; + totalWarnings += result.warnings.length; + } + } + } + + console.log( + `\nDone: ${totalUpdated} updated, ${totalUnchanged} already in sync, ${totalMissing} missing target(s)` + + (totalWarnings > 0 ? `, ${totalWarnings} warning(s)` : "") + ); + + if (dryRun && totalUpdated === 0 && totalMissing === 0) { + console.log("Nothing to sync. Hashes already match English source."); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/.github/scripts/i18n/translation-config.json b/.github/scripts/i18n/translation-config.json index b539eda6a..1db5b5c9d 100644 --- a/.github/scripts/i18n/translation-config.json +++ b/.github/scripts/i18n/translation-config.json @@ -12,8 +12,8 @@ } ], "auto_chunk": { - "min_body_chars": 10000, - "min_sections": 4 + "min_body_chars": 3000, + "min_sections": 2 }, "languages": [ { diff --git a/package.json b/package.json index b0320352c..d2acb6c99 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,8 @@ "translate:snippets:dry-run": "bun .github/scripts/i18n/translate-i18n.ts --snippets --dry-run", "translate:check-truncation": "bun .github/scripts/i18n/check-translation-truncation.ts", "translate:repair-truncated": "bun .github/scripts/i18n/translate-i18n.ts --repair-truncated", + "translate:sync-hash": "bun .github/scripts/i18n/sync-hash-i18n.ts", + "translate:repair-fences": "bun .github/scripts/i18n/repair-fences-i18n.ts", "translate:sync-docs-json": "bun .github/scripts/i18n/translate-i18n.ts --sync-docs-json", "glossary:sync": "bun .github/scripts/i18n/sync-glossary.mjs", "glossary:sync:dry-run": "bun .github/scripts/i18n/sync-glossary.mjs --dry-run", From f419dde1c53b7934b908494a3fd997e2fb864484 Mon Sep 17 00:00:00 2001 From: ComfyUI Wiki Date: Sun, 5 Jul 2026 23:30:16 +0800 Subject: [PATCH 2/2] Fix sync-hash snippet path routing and repair-fences dry-run count. Align file-arg filtering with translate-i18n so snippets/ paths are not processed as pages, and report only actionable fence repairs during dry-run. --- .github/scripts/i18n/README.md | 1 + .github/scripts/i18n/repair-fences-i18n.ts | 4 +++- .github/scripts/i18n/sync-hash-i18n.ts | 25 +++++++++++++--------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/scripts/i18n/README.md b/.github/scripts/i18n/README.md index 35d27f423..0aabe8b32 100644 --- a/.github/scripts/i18n/README.md +++ b/.github/scripts/i18n/README.md @@ -238,6 +238,7 @@ env → `frontend_locales_path` in `translation-config.json` → | `translate-i18n.ts` | translation entry point | | `chunked-translate.ts` | split/reassemble long MDX (`heading_sections`, `update_blocks`) | | `sync-hash-i18n.ts` | Refresh translation hashes after manual edits (no API) | +| `repair-fences-i18n.ts` | Append missing closing ``` in translations (no API) | | `sync-glossary.mjs` | rebuild the glossary frontend mirror | | `glossary.mjs` | load glossary layers, select + inject terms | | `i18n-config.mjs` | shared path rules from `translation-config.json` | diff --git a/.github/scripts/i18n/repair-fences-i18n.ts b/.github/scripts/i18n/repair-fences-i18n.ts index 56fc858b7..b82f8b341 100644 --- a/.github/scripts/i18n/repair-fences-i18n.ts +++ b/.github/scripts/i18n/repair-fences-i18n.ts @@ -73,6 +73,7 @@ async function main() { console.log(`Found ${fenceIssues.length} file(s) with unclosed \`\`\` blocks${dryRun ? " [dry-run]" : ""}:`); + let wouldRepair = 0; let repaired = 0; let skipped = 0; const repairedPairs: { lang: string; enRel: string }[] = []; @@ -101,6 +102,7 @@ async function main() { if (dryRun) { console.log(` [${issue.lang}] would repair: ${issue.targetRel}`); console.log(` ${result.detail}`); + wouldRepair++; } else { await writeFile(targetPath, result.content); console.log(` [${issue.lang}] repaired: ${issue.targetRel}`); @@ -120,7 +122,7 @@ async function main() { } console.log( - `\nDone: ${dryRun ? fenceIssues.length : repaired} repaired, ${skipped} skipped` + + `\nDone: ${dryRun ? wouldRepair : repaired} repaired, ${skipped} skipped` + (fenceIssues.some((i) => i.reasons.includes("missing_code_fence")) && !dryRun ? "\nNote: files with missing_code_fence (fewer fences than EN) may need translate:repair-truncated." : "") diff --git a/.github/scripts/i18n/sync-hash-i18n.ts b/.github/scripts/i18n/sync-hash-i18n.ts index a45432a1a..39eac0317 100644 --- a/.github/scripts/i18n/sync-hash-i18n.ts +++ b/.github/scripts/i18n/sync-hash-i18n.ts @@ -297,15 +297,20 @@ async function syncOneFile( return { status: "updated", warnings }; } +function filterFileArgsForPhase(fileArgs: string[], snippetsMode: boolean): string[] { + return fileArgs + .map((f) => stripLangPrefix(f, config.languages)) + .filter((f) => + snippetsMode + ? isEnglishSnippetPath(f.replace(/^snippets\//, ""), pathFilterOpts) + : isEnglishPagePath(f, pathFilterOpts) + ); +} + async function collectEnglishFiles(snippetsMode: boolean, fileArgs: string[]): Promise { - if (fileArgs.length > 0) { - return fileArgs - .map((f) => stripLangPrefix(f, config.languages)) - .filter((f) => - snippetsMode - ? isEnglishSnippetPath(f.replace(/^snippets\//, ""), pathFilterOpts) - : isEnglishPagePath(f, pathFilterOpts) - ); + const scopedArgs = filterFileArgsForPhase(fileArgs, snippetsMode); + if (scopedArgs.length > 0) { + return scopedArgs; } if (snippetsMode) { @@ -352,8 +357,8 @@ async function main() { } else if (pagesOnly) { phases.push({ snippetsMode: false, fileArgs, label: "pages" }); } else if (fileArgs.length > 0) { - const pageArgs = fileArgs.filter((f) => !f.replace(/^snippets\//, "").startsWith("snippets/")); - const snippetArgs = fileArgs.filter((f) => f.startsWith("snippets/")); + const pageArgs = filterFileArgsForPhase(fileArgs, false); + const snippetArgs = filterFileArgsForPhase(fileArgs, true); if (pageArgs.length > 0) phases.push({ snippetsMode: false, fileArgs: pageArgs, label: "pages" }); if (snippetArgs.length > 0) phases.push({ snippetsMode: true, fileArgs: snippetArgs, label: "snippets" }); if (phases.length === 0) phases.push({ snippetsMode: false, fileArgs, label: "pages" });