diff --git a/README.md b/README.md index 5bd167a7..19095041 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ The general file structure for an extension is as follows: - `src/` contains the source code for the extension - `src/main.ts` is the main entry file for the extension (registers commands and wires interlinear XML) - `src/types/interlinearizer.d.ts` is this extension's types file that defines how other extensions can use this extension through the `papi`. It is copied into the build folder - - `src/parsers/interlinearXmlParser.ts` parses interlinear XML into structured data (uses fast-xml-parser). The PT9 XML schema and parsed output are documented in `src/parsers/pt9-xml.md` + - `src/parsers/pt9/` contains parser and schema for parsing PT9 interlinear XML into structured data - `*.web-view.tsx` files will be treated as React WebViews - `*.web-view.scss` files provide styles for WebViews - `*.web-view.html` files are a conventional way to provide HTML WebViews (no special functionality) diff --git a/__mocks__/interlinearXmlContent.ts b/__mocks__/interlinearXmlContent.ts deleted file mode 100644 index d40e831b..00000000 --- a/__mocks__/interlinearXmlContent.ts +++ /dev/null @@ -1,9 +0,0 @@ -/** - * @file Jest mock for webpack ?raw XML import. Exports the contents of test-data/Interlinear_en_MAT.xml - * so interlinearizer.web-view.tsx can parse it in unit tests without webpack. - */ -import fs from 'fs'; -import path from 'path'; - -const xmlPath = path.join(__dirname, '..', 'test-data', 'Interlinear_en_MAT.xml'); -module.exports = fs.readFileSync(xmlPath, 'utf-8'); diff --git a/__mocks__/papi-backend.ts b/__mocks__/papi-backend.ts index 4f9cab20..c8d3a9f4 100644 --- a/__mocks__/papi-backend.ts +++ b/__mocks__/papi-backend.ts @@ -51,14 +51,6 @@ module.exports = { __esModule: true, default: defaultExport, logger: mockLogger, - __mockRegisterWebViewProvider: mockRegisterWebViewProvider, - __mockRegisterCommand: mockRegisterCommand, - __mockOpenWebView: mockOpenWebView, - __mockSelectProject: mockSelectProject, - __mockGetOpenWebViewDefinition: mockGetOpenWebViewDefinition, - __mockOnDidOpenWebView: mockOnDidOpenWebView, - __mockOnDidCloseWebView: mockOnDidCloseWebView, - __mockLogger: mockLogger, }; /** Marks this file as a module so top-level const/let are module-scoped; avoids TS "redeclare" when both papi-backend and papi-frontend mocks are in the project (they are used mutually exclusively by Jest). */ diff --git a/__mocks__/papi-frontend-react.ts b/__mocks__/papi-frontend-react.ts index a2ba000f..52c6e6c0 100644 --- a/__mocks__/papi-frontend-react.ts +++ b/__mocks__/papi-frontend-react.ts @@ -1,63 +1,70 @@ /** - * @file Jest mock for @papi/frontend/react. Provides stub implementations of various PAPI React hooks so + * @file Jest mock for @papi/frontend/react. Provides stub implementations of PAPI React hooks so * WebView/frontend components can be unit-tested without the real Platform API. */ /** - * useData('providerName') returns an object whose keys are data type names and values are hooks. - * Mock: any property returns a function that returns [undefined, setter, false]. + * Mock for `useProjectData`. Returns a `Proxy` whose property accesses each yield a function + * returning `[undefined, jest.fn(), false]`, matching the real hook's `[data, setter, isLoading]` + * tuple without requiring a live data provider. */ -const createUseDataLikeHook = () => - jest.fn(() => - new Proxy( - {}, - { - get: () => () => [undefined, jest.fn(), false], - }, - ), - ); +const useProjectData = jest.fn(() => + new Proxy( + {}, + { + get: () => () => [undefined, jest.fn(), false], + }, + ), +); -const useDataProvider = jest.fn().mockReturnValue(undefined); -const useData = createUseDataLikeHook(); -const useScrollGroupScrRef = jest.fn().mockReturnValue([undefined, jest.fn()]); -const useSetting = jest.fn().mockImplementation((_key: string, defaultState: unknown) => [defaultState, jest.fn()]); -const useProjectData = createUseDataLikeHook(); -const useProjectDataProvider = jest.fn().mockReturnValue(undefined); +/** + * Mock for `useProjectSetting`. Returns `[defaultState, setSetting, resetSetting, isLoading]`, + * passing `defaultState` through unchanged so callers receive a predictable initial value. + * + * @param _projectDataProviderSource - Ignored project data provider source. + * @param _key - Ignored setting key. + * @param defaultState - Value surfaced as the current setting state. + * @returns Tuple of `[defaultState, jest.fn(), jest.fn(), false]`. + */ const useProjectSetting = jest .fn() - .mockImplementation((_projectInterface: string, _projectIdOrPdp: unknown, _key: string, defaultState: unknown) => [ + .mockImplementation((_projectDataProviderSource: unknown, _key: string, defaultState: unknown) => [ defaultState, jest.fn(), + jest.fn(), + false, ]); -const useDialogCallback = jest.fn().mockReturnValue(jest.fn()); -const useDataProviderMulti = jest.fn().mockReturnValue([]); -/** Returns a map of localization key -> key (so tests get a string for each key). */ -const useLocalizedStrings = jest.fn().mockImplementation((keys: string[]) => - Array.isArray(keys) ? keys.reduce>((acc, k) => ({ ...acc, [k]: k }), {}) : {}, -); -const useWebViewController = jest.fn().mockReturnValue(undefined); -const useRecentScriptureRefs = jest.fn().mockReturnValue([]); + +/** + * Mock for `useLocalizedStrings`. Maps each requested key to itself so tests receive a + * predictable `Record` without a real localization service. + * + * @param keys - BCP47-style string keys to resolve. + * @returns Tuple of `[record, isLoading]` where every key maps to itself and `isLoading` is + * `false`. + */ +const useLocalizedStrings = jest.fn().mockImplementation((keys: string[]) => [ + Array.isArray(keys) ? keys.reduce>((acc, k) => { acc[k] = k; return acc; }, {}) : {}, + false, +]); + +/** + * Mock for `useRecentScriptureRefs`. Returns an empty history and a no-op `addRecentScriptureRef` + * so components that display recent references render without errors. + * + * @returns Object with `recentScriptureRefs` (empty array) and `addRecentScriptureRef` (jest spy). + */ +const useRecentScriptureRefs = jest + .fn() + .mockImplementation(() => ({ recentScriptureRefs: [], addRecentScriptureRef: jest.fn() })); module.exports = { __esModule: true, - useDataProvider, - useData, - useScrollGroupScrRef, - useSetting, useProjectData, - useProjectDataProvider, useProjectSetting, - useDialogCallback, - useDataProviderMulti, useLocalizedStrings, - useWebViewController, useRecentScriptureRefs, - __mockUseDataProvider: useDataProvider, - __mockUseData: useData, - __mockUseLocalizedStrings: useLocalizedStrings, - __mockUseSetting: useSetting, - __mockUseProjectData: useProjectData, - __mockUseProjectDataProvider: useProjectDataProvider, - __mockUseProjectSetting: useProjectSetting, - __mockUseWebViewController: useWebViewController, }; + +/** Marks this file as a module so top-level const/let are module-scoped. */ +export {}; diff --git a/__mocks__/papi-frontend.ts b/__mocks__/papi-frontend.ts index 7536d654..009859da 100644 --- a/__mocks__/papi-frontend.ts +++ b/__mocks__/papi-frontend.ts @@ -1,7 +1,6 @@ /** - * @file Jest mock for @papi/frontend. Provides papi, logger, network, projectDataProviders, and other - * renderer API stubs so WebView/frontend code can be unit-tested without loading the real - * Platform API. + * @file Jest mock for @papi/frontend. Provides a logger stub so WebView/frontend code can be + * unit-tested without loading the real Platform API. */ const mockLogger = { @@ -11,50 +10,10 @@ const mockLogger = { warn: jest.fn(), }; -const mockNetwork = { - request: jest.fn(), - subscribe: jest.fn().mockReturnValue({ dispose: jest.fn() }), -}; - -const mockProjectDataProviders = { - get: jest.fn().mockResolvedValue(undefined), - register: jest.fn().mockResolvedValue({ dispose: jest.fn() }), -}; - -const mockWebViews = { - getWebView: jest.fn(), - openWebView: jest.fn().mockResolvedValue(undefined), -}; - -/** Default papi object shape used in renderer/WebViews. Only commonly used services are stubbed. */ -const papi = { - logger: mockLogger, - network: mockNetwork, - projectDataProviders: mockProjectDataProviders, - webViews: mockWebViews, - react: {}, // Re-export of @papi/frontend/react; tests usually import that module directly. -}; - -const defaultExport = { - ...papi, - __mockLogger: mockLogger, - __mockNetwork: mockNetwork, - __mockProjectDataProviders: mockProjectDataProviders, - __mockWebViews: mockWebViews, -}; - module.exports = { __esModule: true, - default: defaultExport, logger: mockLogger, - network: mockNetwork, - projectDataProviders: mockProjectDataProviders, - webViews: mockWebViews, - __mockLogger: mockLogger, - __mockNetwork: mockNetwork, - __mockProjectDataProviders: mockProjectDataProviders, - __mockWebViews: mockWebViews, }; -/** Marks this file as a module so top-level const/let are module-scoped; avoids TS "redeclare" when both papi-backend and papi-frontend mocks are in the project (they are used mutually exclusively by Jest). */ +/** Marks this file as a module so top-level const/let are module-scoped. */ export {}; diff --git a/__mocks__/platform-bible-react.tsx b/__mocks__/platform-bible-react.tsx new file mode 100644 index 00000000..0266fdd9 --- /dev/null +++ b/__mocks__/platform-bible-react.tsx @@ -0,0 +1,84 @@ +/** + * @file Jest mock for platform-bible-react. The real package ships ESM which Jest cannot parse + * without extra transform configuration. This stub provides the subset used by extension + * components: `BookChapterControl`, `BOOK_CHAPTER_CONTROL_STRING_KEYS`, `TabToolbar`, and + * `ScrollGroupSelector`. + */ + +import type { ReactElement, ReactNode } from 'react'; + +interface SerializedVerseRef { + book: string; + chapterNum: number; + verseNum: number; + verse?: string; + versificationStr?: string; +} + +export const BOOK_CHAPTER_CONTROL_STRING_KEYS: string[] = []; + +export function TabToolbar({ + startAreaChildren, + endAreaChildren, +}: Readonly<{ + className?: string; + startAreaChildren?: ReactNode; + endAreaChildren?: ReactNode; + onSelectProjectMenuItem?: () => void; + onSelectViewInfoMenuItem?: () => void; +}>): ReactElement { + return ( +
+
{startAreaChildren}
+
{endAreaChildren}
+
+ ); +} + +export function ScrollGroupSelector({ + availableScrollGroupIds, + scrollGroupId, + onChangeScrollGroupId, +}: Readonly<{ + availableScrollGroupIds?: (number | undefined)[]; + scrollGroupId?: number; + onChangeScrollGroupId?: (id: number | undefined) => void; +}>): ReactElement { + return ( + + ); +} + +export function BookChapterControl({ + scrRef, + handleSubmit, + onAddRecentSearch, +}: Readonly<{ + scrRef: SerializedVerseRef; + handleSubmit: (ref: SerializedVerseRef) => void; + className?: string; + localizedStrings?: Record; + recentSearches?: SerializedVerseRef[]; + onAddRecentSearch?: (scrRef: SerializedVerseRef) => void; + id?: string; +}>): ReactElement { + return ( +
+ {scrRef.book} {scrRef.chapterNum}:{scrRef.verseNum} + +
+ ); +} diff --git a/__mocks__/platform-bible-utils.ts b/__mocks__/platform-bible-utils.ts index 35990083..34c867fe 100644 --- a/__mocks__/platform-bible-utils.ts +++ b/__mocks__/platform-bible-utils.ts @@ -36,7 +36,7 @@ class UnsubscriberAsyncList { (unsubscriber as { dispose: UnsubscriberFn }).dispose.bind(unsubscriber) ); } else if (typeof unsubscriber === 'function') { - this.unsubscribers.add(unsubscriber as UnsubscriberFn); + this.unsubscribers.add(unsubscriber); } }); } @@ -53,13 +53,16 @@ class UnsubscriberAsyncList { } } -/** Minimal PlatformError shape matching the real platform-bible-utils type. */ +/** + * Minimal PlatformError shape matching the real platform-bible-utils type. Uses `platformErrorVersion` + * as the discriminant — the same field the real `isPlatformError` checks. + */ interface PlatformError { message: string; - isPlatformError: true; + platformErrorVersion: number; } const isPlatformError = (value: unknown): value is PlatformError => - typeof value === 'object' && value !== null && (value as PlatformError).isPlatformError === true; + typeof value === 'object' && value !== null && 'platformErrorVersion' in (value); export { UnsubscriberAsyncList, isPlatformError }; diff --git a/cspell.json b/cspell.json index efd45568..3244aae5 100644 --- a/cspell.json +++ b/cspell.json @@ -22,11 +22,13 @@ "eflomal", "electronmon", "endregion", + "eten", "finalizer", "Fragmenter", "guids", "hopkinson", "iframes", + "imte", "interlinearization", "interlinearizer", "localstorage", @@ -44,6 +46,8 @@ "pdps", "plusplus", "proxied", + "Punct", + "recalc", "reinitializing", "reserialized", "scriptio", diff --git a/jest.config.ts b/jest.config.ts index d4dd6155..ed3741d7 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -23,15 +23,10 @@ const config: Config = { */ collectCoverage: false, - /** - * Collect coverage from parsers, main entry (main.ts), and WebView UI - * (interlinearizer.web-view.tsx). Excludes test files, type declarations, and build artifacts. - */ + /** Collect coverage from all source files. Excludes type declarations and test files. */ collectCoverageFrom: [ - 'src/parsers/**/*.ts', - 'src/main.ts', - 'src/**/*.web-view.tsx', - '!src/parsers/**/*.d.ts', + 'src/**/*.{ts,tsx}', + '!src/**/*.d.ts', '!src/**/__tests__/**', '!src/**/*.test.{ts,tsx}', '!src/**/*.spec.{ts,tsx}', @@ -92,12 +87,12 @@ const config: Config = { '^@papi/frontend/react$': '/__mocks__/papi-frontend-react.ts', /** Mock so test-helpers get UnsubscriberAsyncList without loading ESM deps. */ '^platform-bible-utils$': '/__mocks__/platform-bible-utils.ts', + /** Mock ESM deps that Jest cannot parse. */ + '^platform-bible-react$': '/__mocks__/platform-bible-react.tsx', /** Resolve webpack ?inline imports. */ '^(.+)\\.web-view\\?inline$': '/__mocks__/web-view-inline.ts', /** Resolve webpack ?inline imports: SCSS content. */ '^(.+)\\.(scss|sass|css)\\?inline$': '/__mocks__/styleInlineMock.ts', - /** Resolve webpack ?raw import for test XML in web-view. */ - '^(.+)/Interlinear_en_MAT\\.xml\\?raw$': '/__mocks__/interlinearXmlContent.ts', }, /** Exclude dist from module resolution to avoid Haste naming collision with root package.json. */ diff --git a/package-lock.json b/package-lock.json index ab4c249c..8db7b58d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@tailwindcss/typography": "^0.5.16", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", + "@testing-library/user-event": "^14.6.1", "@types/jest": "^30.0.0", "@types/node": "^22.19.13", "@types/react": "^18.3.18", @@ -2827,6 +2828,20 @@ } } }, + "node_modules/@testing-library/user-event": { + "version": "14.6.1", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.1.tgz", + "integrity": "sha512-vq7fv0rnt+QTXgPxr5Hjc210p6YKq2kmdziLgnsZGgLJ9e6VAShx1pACLuRjd/AS/sr7phAR58OIIpf0LlmQNw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12", + "npm": ">=6" + }, + "peerDependencies": { + "@testing-library/dom": ">=7.21.4" + } + }, "node_modules/@tsconfig/node10": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", diff --git a/package.json b/package.json index dcbd2e8f..30612746 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "@tailwindcss/typography": "^0.5.16", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", + "@testing-library/user-event": "^14.6.1", "@types/jest": "^30.0.0", "@types/node": "^22.19.13", "@types/react": "^18.3.18", diff --git a/src/__tests__/interlinearizer.web-view.test.tsx b/src/__tests__/interlinearizer.web-view.test.tsx index bab23ce8..fd6f65f5 100644 --- a/src/__tests__/interlinearizer.web-view.test.tsx +++ b/src/__tests__/interlinearizer.web-view.test.tsx @@ -5,7 +5,25 @@ import type { WebViewProps } from '@papi/core'; import type { SerializedVerseRef } from '@sillsdev/scripture'; import { render, screen } from '@testing-library/react'; -import { useProjectData } from '@papi/frontend/react'; +import userEvent from '@testing-library/user-event'; +import { + useLocalizedStrings, + useProjectData, + useProjectSetting, + useRecentScriptureRefs, +} from '@papi/frontend/react'; +import type { Book } from 'interlinearizer'; +import { extractBookFromUsj } from 'parsers/papi/usjBookExtractor'; +import { tokenizeBook } from 'parsers/papi/bookTokenizer'; + +jest.mock('parsers/papi/bookTokenizer'); +jest.mock('parsers/papi/usjBookExtractor'); + +/** + * Matches the PlatformError shape from platform-bible-utils (discriminated by + * platformErrorVersion). + */ +type PlatformError = { platformErrorVersion: number; message: string }; /** * Load the WebView module; it assigns the component to globalThis.webViewComponent. This pattern is @@ -24,8 +42,106 @@ const defaultScrRef: SerializedVerseRef = { book: 'GEN', chapterNum: 1, verseNum const testProjectId = 'test-project-id'; +/** Pre-built Book with one GEN 1:1 segment — used by tests that need the strip to render. */ +const GEN_1_1_BOOK: Book = { + id: 'GEN', + bookRef: 'GEN', + textVersion: 'v1', + segments: [ + { + id: 'GEN 1:1', + startRef: { book: 'GEN', chapter: 1, verse: 1 }, + endRef: { book: 'GEN', chapter: 1, verse: 1 }, + baselineText: 'In the beginning.', + tokens: [ + { + id: 'GEN 1:1:0', + surfaceText: 'In', + writingSystem: 'en', + type: 'word', + charStart: 0, + charEnd: 2, + }, + ], + }, + ], +}; + +/** Pre-built Book with no segments — used by the no-verse-data test. */ +const GEN_EMPTY_BOOK: Book = { id: 'GEN', bookRef: 'GEN', textVersion: 'v1', segments: [] }; + +/** Book with two segments in GEN 1 — used by chapter-display tests. */ +const GEN_1_MULTI_BOOK: Book = { + id: 'GEN', + bookRef: 'GEN', + textVersion: 'v1', + segments: [ + { + id: 'GEN 1:1', + startRef: { book: 'GEN', chapter: 1, verse: 1 }, + endRef: { book: 'GEN', chapter: 1, verse: 1 }, + baselineText: 'In the beginning.', + tokens: [ + { + id: 'GEN 1:1:0', + surfaceText: 'In', + writingSystem: 'en', + type: 'word', + charStart: 0, + charEnd: 2, + }, + ], + }, + { + id: 'GEN 1:2', + startRef: { book: 'GEN', chapter: 1, verse: 2 }, + endRef: { book: 'GEN', chapter: 1, verse: 2 }, + baselineText: 'And the earth.', + tokens: [ + { + id: 'GEN 1:2:0', + surfaceText: 'And', + writingSystem: 'en', + type: 'word', + charStart: 0, + charEnd: 3, + }, + ], + }, + ], +}; + +/** Book with a non-word (punctuation) token — exercises the non-word chip branch. */ +const GEN_1_1_PUNCTUATION_BOOK: Book = { + id: 'GEN', + bookRef: 'GEN', + textVersion: 'v1', + segments: [ + { + id: 'GEN 1:1', + startRef: { book: 'GEN', chapter: 1, verse: 1 }, + endRef: { book: 'GEN', chapter: 1, verse: 1 }, + baselineText: '.', + tokens: [ + { + id: 'GEN 1:1:0', + surfaceText: '.', + writingSystem: 'en', + type: 'punctuation', + charStart: 0, + charEnd: 1, + }, + ], + }, + ], +}; + /** Builds a minimal WebViewProps for tests. */ -function makeProps(projectId?: string): WebViewProps { +function makeProps( + projectId?: string, + scrRef: SerializedVerseRef = defaultScrRef, + setScrRef: (r: SerializedVerseRef) => void = () => {}, +): WebViewProps { return { id: 'test-id', webViewType: 'interlinearizer.mainWebView', @@ -40,7 +156,7 @@ function makeProps(projectId?: string): WebViewProps { (r: SerializedVerseRef) => void, number | undefined, (id: number | undefined) => void, - ] => [defaultScrRef, () => {}, undefined, () => {}], + ] => [scrRef, setScrRef, undefined, () => {}], updateWebViewDefinition: () => true, }; } @@ -52,15 +168,33 @@ function mockBookData(value: unknown, isLoading = false): void { })); } +/** Configures useProjectSetting to return the given writing system tag. */ +function mockWritingSystem(tag: string | PlatformError = 'en'): void { + jest.mocked(useProjectSetting).mockReturnValue([tag, jest.fn(), jest.fn(), false]); +} + describe('InterlinearizerWebView', () => { beforeEach(() => { mockBookData(undefined); + mockWritingSystem(); + jest.mocked(useLocalizedStrings).mockReturnValue([{}, false]); + jest.mocked(useRecentScriptureRefs).mockReturnValue({ + recentScriptureRefs: [], + addRecentScriptureRef: jest.fn(), + }); + jest.mocked(extractBookFromUsj).mockReturnValue({ + bookCode: 'GEN', + writingSystem: 'en', + contentHash: 'abc', + verses: [], + }); + jest.mocked(tokenizeBook).mockReturnValue(GEN_1_1_BOOK); }); - it('renders the heading "Interlinearizer"', () => { + it('shows the book chapter control regardless of whether a project is linked', () => { render(); - expect(screen.getByRole('heading', { name: /interlinearizer/i })).toBeInTheDocument(); + expect(screen.getByTestId('book-chapter-control')).toBeInTheDocument(); }); it('shows a prompt to open from a project when no projectId is provided', () => { @@ -69,12 +203,12 @@ describe('InterlinearizerWebView', () => { expect(screen.getByText(/open this webview from a paratext project/i)).toBeInTheDocument(); }); - it('shows the book and projectId when a project is linked', () => { - mockBookData({ type: 'USJ', version: '3.1', content: [] }); + it('shows the book chapter control and renders a segment when a project is linked', () => { + mockBookData({}); render(); - expect(screen.getByText(new RegExp(testProjectId))).toBeInTheDocument(); - expect(screen.getByText(/GEN · project/)).toBeInTheDocument(); + expect(screen.getByTestId('book-chapter-control')).toBeInTheDocument(); + expect(screen.getByText('In')).toBeInTheDocument(); }); it('shows Loading when projectId is set but book data has not arrived', () => { @@ -92,22 +226,148 @@ describe('InterlinearizerWebView', () => { expect(screen.getByText(/no usj book available for gen in project/i)).toBeInTheDocument(); }); - it('shows the raw USFM when book data arrives', () => { - mockBookData({ - type: 'USJ', - version: '3.1', - content: [{ type: 'book', marker: 'id', code: 'EXO' }], - }); + it('renders token chips when the tokenized book has a segment for the current reference', () => { + mockBookData({}); render(); - expect(screen.getByText(/"code": "EXO"/)).toBeInTheDocument(); + expect(screen.getByText('In')).toBeInTheDocument(); + }); + + it('shows a no-verse message when the tokenized book has no segments at all', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockReturnValue(GEN_EMPTY_BOOK); + render(); + + expect(screen.getByText(/no verse data for gen 1\./i)).toBeInTheDocument(); + }); + + it('renders all segments in the current chapter', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockReturnValue(GEN_1_MULTI_BOOK); + render(); + + expect(screen.getByText('In')).toBeInTheDocument(); + expect(screen.getByText('And')).toBeInTheDocument(); + }); + + it('highlights only the segment matching the current verse', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockReturnValue(GEN_1_MULTI_BOOK); + // defaultScrRef is GEN 1:1, so verse 1 is active + const { container } = render(); + + const activeSegments = container.querySelectorAll('button[aria-current="true"]'); + expect(activeSegments).toHaveLength(1); + }); + + it('shows all chapter segments when navigating to a title reference (verse 0)', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockReturnValue(GEN_1_MULTI_BOOK); + const titleRef: SerializedVerseRef = { book: 'GEN', chapterNum: 1, verseNum: 0 }; + render(); + + expect(screen.getByText('In')).toBeInTheDocument(); + expect(screen.getByText('And')).toBeInTheDocument(); }); it('shows an error heading and message when book data is a PlatformError', () => { - mockBookData({ isPlatformError: true, message: 'Project not found' }); + mockBookData({ platformErrorVersion: 1, message: 'Project not found' }); render(); expect(screen.getByRole('heading', { name: /error loading book/i })).toBeInTheDocument(); expect(screen.getByText(/project not found/i)).toBeInTheDocument(); }); + + it('falls back to "und" writing system when useProjectSetting returns a PlatformError', () => { + mockBookData({}); + mockWritingSystem({ platformErrorVersion: 1, message: 'Setting unavailable' }); + render(); + + expect(screen.getByText('In')).toBeInTheDocument(); + expect(extractBookFromUsj).toHaveBeenCalledWith(expect.anything(), 'und'); + }); + + it('falls back to "und" writing system when useProjectSetting returns an empty string', () => { + mockBookData({}); + mockWritingSystem(''); + render(); + + expect(screen.getByText('In')).toBeInTheDocument(); + expect(extractBookFromUsj).toHaveBeenCalledWith(expect.anything(), 'und'); + }); + + it('shows an error heading and message when tokenization throws an Error', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockImplementation(() => { + throw new Error('parse failure'); + }); + render(); + + expect(screen.getByRole('heading', { name: /error processing book/i })).toBeInTheDocument(); + expect(screen.getByText('parse failure')).toBeInTheDocument(); + }); + + it('shows an error message when tokenization throws a non-Error value', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockImplementation(() => { + // eslint-disable-next-line no-throw-literal + throw 'unexpected string error'; + }); + render(); + + expect(screen.getByRole('heading', { name: /error processing book/i })).toBeInTheDocument(); + expect(screen.getByText('unexpected string error')).toBeInTheDocument(); + }); + + it('renders non-word tokens as muted chips', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockReturnValue(GEN_1_1_PUNCTUATION_BOOK); + render(); + + expect(screen.getByText('.')).toBeInTheDocument(); + }); + + it('calls setScrRef and addRecentScriptureRef when the verse picker submits', async () => { + mockBookData({}); + const mockSetScrRef = jest.fn(); + const mockAddRecentRef = jest.fn(); + jest.mocked(useRecentScriptureRefs).mockReturnValue({ + recentScriptureRefs: [], + addRecentScriptureRef: mockAddRecentRef, + }); + render(); + + await userEvent.click(screen.getByRole('button', { name: /submit reference/i })); + + expect(mockSetScrRef).toHaveBeenCalledWith(defaultScrRef); + expect(mockAddRecentRef).toHaveBeenCalledWith(defaultScrRef); + }); + + it('calls setScrRef with the segment ref when a verse box is clicked', async () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockReturnValue(GEN_1_MULTI_BOOK); + const mockSetScrRef = jest.fn(); + // Start at verse 1; click verse 2's token to select it + render(); + + await userEvent.click(screen.getByText('And')); + + expect(mockSetScrRef).toHaveBeenCalledWith({ book: 'GEN', chapterNum: 1, verseNum: 2 }); + }); + + it('passes a book-stable ref to BookUSJ so chapter and verse changes do not re-fetch the book', () => { + const mockBookUSJ = jest.fn().mockReturnValue([{}, jest.fn(), false]); + jest.mocked(useProjectData).mockImplementation(() => ({ BookUSJ: mockBookUSJ })); + const { rerender } = render(); + rerender( + , + ); + + const refsPassed = mockBookUSJ.mock.calls.map((c) => c[0]); + refsPassed.forEach((ref) => expect(ref).toEqual({ book: 'GEN', chapterNum: 1, verseNum: 1 })); + expect(mockBookUSJ.mock.calls.length).toBeGreaterThanOrEqual(2); + refsPassed.slice(1).forEach((ref) => expect(ref).toBe(refsPassed[0])); + }); }); diff --git a/src/__tests__/main.test.ts b/src/__tests__/main.test.ts index 8cdf744d..b99af479 100644 --- a/src/__tests__/main.test.ts +++ b/src/__tests__/main.test.ts @@ -50,6 +50,41 @@ const { __mockLogger, } = papiBackendMock; +function isCallable(f: unknown): f is (...args: unknown[]) => unknown { + return typeof f === 'function'; +} + +function findRegisteredHandler(commandName: string): ((...args: unknown[]) => unknown) | undefined { + const call = jest.mocked(__mockRegisterCommand).mock.calls.find((c) => c[0] === commandName); + const rawHandler: unknown = call?.[1]; + return isCallable(rawHandler) ? rawHandler : undefined; +} + +async function getOpenForWebViewHandler(): Promise< + (webViewId?: string) => Promise +> { + const context = createTestActivationContext(); + await activate(context); + const rawHandler = findRegisteredHandler('interlinearizer.openForWebView'); + if (!rawHandler) throw new Error('Handler not found for interlinearizer.openForWebView'); + return async (webViewId?: string): Promise => { + const result: unknown = await rawHandler(webViewId); + return typeof result === 'string' ? result : undefined; + }; +} + +function getOpenWebViewCallback(): (event: { webView: SavedWebViewDefinition }) => void { + const cb: unknown = __mockOnDidOpenWebView.mock.calls[0]?.[0]; + if (!isCallable(cb)) throw new Error('onDidOpenWebView callback not found'); + return (event) => cb(event); +} + +function getCloseWebViewCallback(): (event: { webView: SavedWebViewDefinition }) => void { + const cb: unknown = __mockOnDidCloseWebView.mock.calls[0]?.[0]; + if (!isCallable(cb)) throw new Error('onDidCloseWebView callback not found'); + return (event) => cb(event); +} + describe('main', () => { const mainWebViewType = 'interlinearizer.mainWebView'; @@ -216,32 +251,7 @@ describe('main', () => { }); }); - function isCallable(f: unknown): f is (...args: unknown[]) => unknown { - return typeof f === 'function'; - } - - function findRegisteredHandler( - commandName: string, - ): ((...args: unknown[]) => unknown) | undefined { - const call = jest.mocked(__mockRegisterCommand).mock.calls.find((c) => c[0] === commandName); - const rawHandler: unknown = call?.[1]; - return isCallable(rawHandler) ? rawHandler : undefined; - } - describe('interlinearizer.openForWebView command', () => { - async function getOpenForWebViewHandler(): Promise< - (webViewId?: string) => Promise - > { - const context = createTestActivationContext(); - await activate(context); - const rawHandler = findRegisteredHandler('interlinearizer.openForWebView'); - if (!rawHandler) throw new Error('Handler not found for interlinearizer.openForWebView'); - return async (webViewId?: string): Promise => { - const result: unknown = await rawHandler(webViewId); - return typeof result === 'string' ? result : undefined; - }; - } - it('looks up the projectId from the given WebView and opens the Interlinearizer', async () => { __mockGetOpenWebViewDefinition.mockResolvedValue({ id: 'some-webview', @@ -340,18 +350,6 @@ describe('main', () => { expect(__mockOnDidCloseWebView).toHaveBeenCalledTimes(1); }); - function getOpenWebViewCallback(): (event: { webView: SavedWebViewDefinition }) => void { - const cb: unknown = __mockOnDidOpenWebView.mock.calls[0]?.[0]; - if (!isCallable(cb)) throw new Error('onDidOpenWebView callback not found'); - return (event) => cb(event); - } - - function getCloseWebViewCallback(): (event: { webView: SavedWebViewDefinition }) => void { - const cb: unknown = __mockOnDidCloseWebView.mock.calls[0]?.[0]; - if (!isCallable(cb)) throw new Error('onDidCloseWebView callback not found'); - return (event) => cb(event); - } - describe('onDidOpenWebView callback', () => { it('adds the webView to the project map so subsequent opens reuse the existing tab', async () => { __mockSelectProject.mockResolvedValue('my-project'); diff --git a/src/__tests__/parsers/papi/bookTokenizer.test.ts b/src/__tests__/parsers/papi/bookTokenizer.test.ts new file mode 100644 index 00000000..d36238fd --- /dev/null +++ b/src/__tests__/parsers/papi/bookTokenizer.test.ts @@ -0,0 +1,299 @@ +/** @file Unit tests for {@link tokenizeBook}. */ +/// + +import { tokenizeBook } from 'parsers/papi/bookTokenizer'; +import type { RawBook } from 'parsers/papi/usjBookExtractor'; + +function makeRawBook(verses: { sid: string; text: string }[]): RawBook { + return { bookCode: 'GEN', writingSystem: 'en', contentHash: 'abc123', verses }; +} + +describe('tokenizeBook', () => { + it('maps bookCode and contentHash onto the Book', () => { + const raw = makeRawBook([]); + const book = tokenizeBook(raw); + expect(book.bookRef).toBe('GEN'); + expect(book.id).toBe('GEN'); + expect(book.textVersion).toBe('abc123'); + }); + + it('produces no segments when there are no verses', () => { + expect(tokenizeBook(makeRawBook([])).segments).toEqual([]); + }); + + it('produces one segment per verse in order', () => { + const raw = makeRawBook([ + { sid: 'GEN 1:1', text: 'First.' }, + { sid: 'GEN 1:2', text: 'Second.' }, + ]); + const { segments } = tokenizeBook(raw); + expect(segments).toHaveLength(2); + expect(segments[0].id).toBe('GEN 1:1'); + expect(segments[1].id).toBe('GEN 1:2'); + }); + + it('sets baselineText to the raw verse text', () => { + const text = 'In the beginning God created the heavens and the earth.'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].baselineText).toBe(text); + }); + + it('sets startRef and endRef from the verse SID', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'Hello.' }])); + expect(segments[0].startRef).toEqual({ book: 'GEN', chapter: 1, verse: 1 }); + expect(segments[0].endRef).toEqual({ book: 'GEN', chapter: 1, verse: 1 }); + }); + + it('upholds the charStart/charEnd invariant for every token', () => { + const text = 'In the beginning, God created.'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + segments[0].tokens.forEach((token) => + expect(text.slice(token.charStart, token.charEnd)).toBe(token.surfaceText), + ); + }); + + it('labels word tokens as word', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'Hello world' }])); + const { tokens } = segments[0]; + expect(tokens.length).toBeGreaterThan(0); + expect(tokens.every((t) => t.type === 'word')).toBe(true); + }); + + it('labels punctuation tokens as punctuation', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: '., ;!' }])); + const { tokens } = segments[0]; + expect(tokens.length).toBeGreaterThan(0); + expect(tokens.every((t) => t.type === 'punctuation')).toBe(true); + }); + + it('produces mixed word and punctuation tokens in the correct order', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'Hello, world.' }])); + const types = segments[0].tokens.map((t) => t.type); + expect(types).toEqual(['word', 'punctuation', 'word', 'punctuation']); + }); + + it('does not produce tokens for whitespace', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: ' ' }])); + expect(segments[0].tokens).toEqual([]); + }); + + it('assigns unique IDs within a segment', () => { + const text = 'A B C.'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + const ids = segments[0].tokens.map((t) => t.id); + expect(new Set(ids).size).toBe(ids.length); + }); + + it('assigns unique IDs across segments', () => { + const raw = makeRawBook([ + { sid: 'GEN 1:1', text: 'Word.' }, + { sid: 'GEN 1:2', text: 'Word.' }, + ]); + const book = tokenizeBook(raw); + // Each token id must start with its segment's id (the SID). + book.segments.forEach((s) => { + s.tokens.forEach((t) => { + expect(t.id.startsWith(s.id)).toBe(true); + }); + }); + // All token ids across all segments must be globally unique. + const ids = book.segments.flatMap((s) => s.tokens.map((t) => t.id)); + expect(new Set(ids).size).toBe(ids.length); + }); + + it('assigns writingSystem to every token', () => { + const raw: RawBook = { + ...makeRawBook([{ sid: 'GEN 1:1', text: 'Hello.' }]), + writingSystem: 'kmr', + }; + const { segments } = tokenizeBook(raw); + const { tokens } = segments[0]; + expect(tokens.length).toBeGreaterThan(0); + expect(tokens.every((t) => t.writingSystem === 'kmr')).toBe(true); + }); + + it('produces an empty token list for an empty verse', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: '' }])); + expect(segments[0].tokens).toEqual([]); + }); + + it('handles Unicode letters (non-ASCII word characters)', () => { + const text = 'Ελληνικά.'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + const wordTokens = segments[0].tokens.filter((t) => t.type === 'word'); + const punctTokens = segments[0].tokens.filter((t) => t.type === 'punctuation'); + expect(wordTokens).toHaveLength(1); + expect(wordTokens[0].surfaceText).toBe('Ελληνικά'); + expect(punctTokens).toHaveLength(1); + expect(punctTokens[0].surfaceText).toBe('.'); + expect(punctTokens[0].type).toBe('punctuation'); + }); + + it('treats a combining-mark sequence as a single word token', () => { + // 'ñ' is the letter n followed by a combining tilde (U+0303). + // The \p{M} branch of TOKEN_RE must match the combining mark so the whole + // sequence is captured as one token rather than split. + const text = 'ñ'; + expect(text.length).toBe(2); // n (U+006E) + combining tilde (U+0303) + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe('ñ'); + }); + + it('throws when a verse SID book code does not match rawBook.bookCode', () => { + const raw: RawBook = { ...makeRawBook([{ sid: 'EXO 1:1', text: 'text' }]), bookCode: 'GEN' }; + expect(() => tokenizeBook(raw)).toThrow( + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining('does not match book code'), + }), + ); + }); + + it('throws on an invalid verse SID', () => { + expect(() => tokenizeBook(makeRawBook([{ sid: 'not-a-ref', text: 'text' }]))).toThrow( + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining('Invalid verse SID'), + }), + ); + }); + + describe('word-internal joiners', () => { + it("tokenizes don't (ASCII apostrophe) as a single word token", () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: "don't" }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe("don't"); + }); + + it('tokenizes don’t (U+2019 right single quote) as a single word token', () => { + const text = 'don’t'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); + + it("tokenizes l'homme as a single word token", () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: "l'homme" }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe("l'homme"); + }); + + it('tokenizes well-known as a single word token', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'well-known' }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe('well-known'); + }); + + it('tokenizes "it\'s well-known" as two word tokens', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: "it's well-known" }])); + const wordTokens = segments[0].tokens.filter((t) => t.type === 'word'); + expect(wordTokens).toHaveLength(2); + expect(wordTokens[0].surfaceText).toBe("it's"); + expect(wordTokens[1].surfaceText).toBe('well-known'); + }); + + it("tokenizes 'hello' as a single word token (both leading and trailing apostrophes absorbed)", () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: "'hello'" }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe("'hello'"); + }); + + it('tokenizes a standalone apostrophe as punctuation (no following word character)', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: "'" }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('punctuation'); + }); + + it("tokenizes word-initial U+0027 as part of the word (e.g. Hebrew aleph romanisation 'Elohim)", () => { + const text = "'Elohim"; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); + + it('tokenizes word-initial U+2019 as part of the word', () => { + const text = '’Elohim'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); + + it("tokenizes word-final U+0027 as part of the word (e.g. Hebrew aleph romanisation bara')", () => { + const text = "bara'"; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); + + it('tokenizes word-final U+2019 as part of the word (e.g. Hebrew aleph romanisation bara’)', () => { + const text = 'bara’'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); + it('tokenizes U+02BC (modifier letter apostrophe) as a word character regardless of position', () => { + // U+02BC is \p{L} so it is inherently a word character — no special handling needed. + const text = 'ʼelohim'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); + + it('tokenizes end- as word then punctuation (trailing joiner is not absorbed)', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'end-' }])); + expect(segments[0].tokens).toHaveLength(2); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe('end'); + expect(segments[0].tokens[1].type).toBe('punctuation'); + expect(segments[0].tokens[1].surfaceText).toBe('-'); + }); + + it('tokenizes -start as punctuation then word (leading joiner is not absorbed)', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: '-start' }])); + expect(segments[0].tokens).toHaveLength(2); + expect(segments[0].tokens[0].type).toBe('punctuation'); + expect(segments[0].tokens[0].surfaceText).toBe('-'); + expect(segments[0].tokens[1].type).toBe('word'); + expect(segments[0].tokens[1].surfaceText).toBe('start'); + }); + + // Double joiners between word chars are absorbed greedily: a--b → one token "a--b". + it('tokenizes a--b as a single word token (greedy double-joiner absorption)', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'a--b' }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe('a--b'); + }); + + it('upholds the charStart/charEnd invariant for joiner-containing tokens', () => { + const text = "it's well-known, don’t you think?"; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + segments[0].tokens.forEach((token) => + expect(text.slice(token.charStart, token.charEnd)).toBe(token.surfaceText), + ); + }); + }); + + it('classifies astral-plane letters (surrogate pairs) as word tokens', () => { + // Gothic letters U+10330–U+1034F are outside the BMP; each code point is two UTF-16 code + // units. Testing surfaceText[0] (a lone surrogate) against WORD_CONTAIN_RE would fail — the + // fix is to test the full surfaceText string. + const text = '𐌰𐌱𐌲'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); +}); diff --git a/src/__tests__/parsers/papi/usjBookExtractor.test.ts b/src/__tests__/parsers/papi/usjBookExtractor.test.ts new file mode 100644 index 00000000..5e819f77 --- /dev/null +++ b/src/__tests__/parsers/papi/usjBookExtractor.test.ts @@ -0,0 +1,279 @@ +/** @file Unit tests for {@link extractBookFromUsj}. */ +/// + +import { extractBookFromUsj, type UsjDocument } from 'parsers/papi/usjBookExtractor'; + +const WS = 'en'; + +describe('extractBookFromUsj', () => { + it('extracts bookCode from the book marker', () => { + const usj: UsjDocument = { + content: [{ type: 'book', code: 'GEN', content: [] }], + }; + expect(extractBookFromUsj(usj, WS).bookCode).toBe('GEN'); + }); + + it('sets writingSystem from the parameter', () => { + const usj: UsjDocument = { + content: [{ type: 'book', code: 'GEN', content: [] }], + }; + expect(extractBookFromUsj(usj, 'kmr').writingSystem).toBe('kmr'); + }); + + it('produces a stable contentHash for identical content', () => { + const a: UsjDocument = { content: [{ type: 'book', code: 'GEN', content: [] }] }; + const b: UsjDocument = { content: [...a.content] }; + expect(extractBookFromUsj(a, 'en').contentHash).toBe(extractBookFromUsj(b, 'es').contentHash); + }); + + it('produces different contentHashes for different content', () => { + const a: UsjDocument = { content: [{ type: 'book', code: 'GEN', content: [] }] }; + const b: UsjDocument = { content: [{ type: 'book', code: 'MAT', content: [] }] }; + expect(extractBookFromUsj(a, WS).contentHash).not.toBe(extractBookFromUsj(b, WS).contentHash); + }); + + it('returns empty verses when there are no verse markers', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { type: 'chapter', number: '1', sid: 'GEN 1' }, + ], + }; + expect(extractBookFromUsj(usj, WS).verses).toEqual([]); + }); + + it('extracts a single verse with its text', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { + type: 'para', + marker: 'p', + content: [ + { type: 'verse', sid: 'GEN 1:1' }, + 'In the beginning God created the heavens and the earth.', + ], + }, + ], + }; + const result = extractBookFromUsj(usj, WS); + expect(result.verses).toHaveLength(1); + expect(result.verses[0]).toEqual({ + sid: 'GEN 1:1', + text: 'In the beginning God created the heavens and the earth.', + }); + }); + + it('extracts multiple verses in document order', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { + type: 'para', + marker: 'p', + content: [ + { type: 'verse', sid: 'GEN 1:1' }, + 'First verse text.', + { type: 'verse', sid: 'GEN 1:2' }, + 'Second verse text.', + ], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(2); + expect(verses[0]).toEqual({ sid: 'GEN 1:1', text: 'First verse text.' }); + expect(verses[1]).toEqual({ sid: 'GEN 1:2', text: 'Second verse text.' }); + }); + + it('accumulates text across multiple paragraphs within a verse', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'PSA', content: [] }, + { + type: 'para', + marker: 'q1', + content: [{ type: 'verse', sid: 'PSA 1:1' }, 'Blessed is the man'], + }, + { + type: 'para', + marker: 'q2', + content: ['who walks not in the counsel of the wicked.'], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(1); + expect(verses[0].text).toBe('Blessed is the man who walks not in the counsel of the wicked.'); + }); + + it('includes text inside inline char nodes', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'JHN', content: [] }, + { + type: 'para', + marker: 'p', + content: [ + { type: 'verse', sid: 'JHN 1:1' }, + 'In the beginning was the ', + { type: 'char', marker: 'nd', content: ['Word'] }, + '.', + ], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses[0].text).toBe('In the beginning was the Word.'); + }); + + it('excludes note content from verse text', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'MAT', content: [] }, + { + type: 'para', + marker: 'p', + content: [ + { type: 'verse', sid: 'MAT 1:1' }, + 'The book of the genealogy', + { type: 'note', marker: 'f', content: ['Some footnote text.'] }, + ' of Jesus Christ.', + ], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses[0].text).toBe('The book of the genealogy of Jesus Christ.'); + }); + + it('produces an empty-text RawVerse when a verse marker has no following text', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { + type: 'para', + marker: 'p', + content: [ + { type: 'verse', sid: 'GEN 1:1' }, + // no text before the next verse + { type: 'verse', sid: 'GEN 1:2' }, + 'Some text.', + ], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(2); + expect(verses[0]).toEqual({ sid: 'GEN 1:1', text: '' }); + expect(verses[1]).toEqual({ sid: 'GEN 1:2', text: 'Some text.' }); + }); + + it('captures text nested directly inside a verse node', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { + type: 'para', + marker: 'p', + content: [{ type: 'verse', sid: 'GEN 1:1', content: ['Inline verse content.'] }], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(1); + expect(verses[0]).toEqual({ sid: 'GEN 1:1', text: 'Inline verse content.' }); + }); + + it('throws when a verse marker is missing its sid attribute', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { type: 'para', marker: 'p', content: [{ type: 'verse' }] }, + ], + }; + expect(() => extractBookFromUsj(usj, WS)).toThrow( + 'verse marker missing required sid attribute', + ); + }); + + it('throws when no book marker with a code attribute is found', () => { + const usj: UsjDocument = { content: [{ type: 'para', content: ['Some text.'] }] }; + expect(() => extractBookFromUsj(usj, WS)).toThrow('no book marker'); + }); + + it('flushes an open verse when a chapter boundary is crossed', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { + type: 'para', + marker: 'p', + content: [{ type: 'verse', sid: 'GEN 1:31' }, 'Last verse of chapter one.'], + }, + { type: 'chapter', number: '2', sid: 'GEN 2' }, + { + type: 'para', + marker: 'p', + content: [{ type: 'verse', sid: 'GEN 2:1' }, 'First verse of chapter two.'], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(2); + expect(verses[0]).toEqual({ sid: 'GEN 1:31', text: 'Last verse of chapter one.' }); + expect(verses[1]).toEqual({ sid: 'GEN 2:1', text: 'First verse of chapter two.' }); + }); + + it('traverses content nested directly inside a chapter node', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { + type: 'chapter', + number: '1', + sid: 'GEN 1', + content: [ + { + type: 'para', + marker: 'p', + content: [{ type: 'verse', sid: 'GEN 1:1' }, 'In the beginning.'], + }, + ], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(1); + expect(verses[0]).toEqual({ sid: 'GEN 1:1', text: 'In the beginning.' }); + }); + + it('skips content of heading para markers encountered inside a verse', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'PSA', content: [] }, + { + type: 'para', + marker: 'p', + content: [{ type: 'verse', sid: 'PSA 119:176' }, 'I have gone astray'], + }, + { type: 'para', marker: 's1', content: ['A section heading'] }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(1); + expect(verses[0].text).toBe('I have gone astray'); + }); + + it('produces a stable contentHash when a node has an optional property explicitly set to undefined', () => { + const withUndefined: UsjDocument = { + content: [{ type: 'book', code: 'GEN', marker: undefined, content: [] }], + }; + const withoutUndefined: UsjDocument = { + content: [{ type: 'book', code: 'GEN', content: [] }], + }; + + const hash = extractBookFromUsj(withUndefined, WS).contentHash; + expect(hash).toBe(extractBookFromUsj(withoutUndefined, WS).contentHash); + }); +}); diff --git a/src/__tests__/parsers/interlinearXmlParser.test.ts b/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts similarity index 90% rename from src/__tests__/parsers/interlinearXmlParser.test.ts rename to src/__tests__/parsers/pt9/interlinearXmlParser.test.ts index 875789f1..c80befcf 100644 --- a/src/__tests__/parsers/interlinearXmlParser.test.ts +++ b/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts @@ -1,10 +1,10 @@ /** @file Unit tests for {@link InterlinearXmlParser}. */ /// -import * as fs from 'fs'; -import * as path from 'path'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; -import { InterlinearXmlParser } from 'parsers/interlinearXmlParser'; +import { InterlinearXmlParser } from 'parsers/pt9/interlinearXmlParser'; describe('InterlinearXmlParser', () => { let parser: InterlinearXmlParser; @@ -578,7 +578,15 @@ describe('InterlinearXmlParser', () => { }); it('parses real test-data file without throwing', () => { - const xmlPath = path.join(__dirname, '..', '..', '..', 'test-data', 'Interlinear_en_MAT.xml'); + const xmlPath = path.join( + __dirname, + '..', + '..', + '..', + '..', + 'test-data', + 'Interlinear_en_MAT.xml', + ); const xml = fs.readFileSync(xmlPath, 'utf-8'); const result = parser.parse(xml); @@ -622,7 +630,12 @@ describe('InterlinearXmlParser', () => { `; - expect(() => parser.parse(xml)).toThrow('Invalid XML: Missing InterlinearData root element'); + expect(() => parser.parse(xml)).toThrow( + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining('Invalid XML: Missing InterlinearData root element'), + }), + ); }); it('throws when GlossLanguage is missing', () => { @@ -637,7 +650,12 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xml)).toThrow( - 'Invalid XML: Missing required attributes GlossLanguage or BookId', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining( + 'Invalid XML: Missing required attributes GlossLanguage or BookId', + ), + }), ); }); @@ -653,7 +671,12 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xml)).toThrow( - 'Invalid XML: Missing required attributes GlossLanguage or BookId', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining( + 'Invalid XML: Missing required attributes GlossLanguage or BookId', + ), + }), ); }); @@ -669,7 +692,12 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xml)).toThrow( - 'Invalid XML: Missing required attributes GlossLanguage or BookId', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining( + 'Invalid XML: Missing required attributes GlossLanguage or BookId', + ), + }), ); }); @@ -678,7 +706,12 @@ describe('InterlinearXmlParser', () => { `; - expect(() => parser.parse(xml)).toThrow('Invalid XML: Missing Verses element'); + expect(() => parser.parse(xml)).toThrow( + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining('Invalid XML: Missing Verses element'), + }), + ); }); it('throws when Cluster is missing Range element', () => { @@ -697,7 +730,10 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xml)).toThrow( - 'Invalid XML: Cluster missing required Range element', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining('Invalid XML: Cluster missing required Range element'), + }), ); }); @@ -718,7 +754,12 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xmlNoIndex)).toThrow( - 'Invalid XML: Range missing required Index or Length attributes', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining( + 'Invalid XML: Range missing or invalid Index/Length attributes (must be non-negative integers)', + ), + }), ); const xmlNoLength = ` @@ -737,7 +778,12 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xmlNoLength)).toThrow( - 'Invalid XML: Range missing required Index or Length attributes', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining( + 'Invalid XML: Range missing or invalid Index/Length attributes (must be non-negative integers)', + ), + }), ); }); @@ -757,7 +803,12 @@ describe('InterlinearXmlParser', () => { `; - expect(() => parser.parse(xml)).toThrow('Invalid XML: Lexeme missing required Id attribute'); + expect(() => parser.parse(xml)).toThrow( + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining('Invalid XML: Lexeme missing required Id attribute'), + }), + ); }); it('throws when the same verse reference appears in more than one item', () => { @@ -795,7 +846,12 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xml)).toThrow( - 'Invalid XML: Duplicate verse reference "MAT 1:1". At most one VerseData per reference is allowed.', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining( + 'Invalid XML: Duplicate verse reference "MAT 1:1". At most one VerseData per reference is allowed.', + ), + }), ); }); }); diff --git a/src/interlinearizer.web-view.tsx b/src/interlinearizer.web-view.tsx index 7eb9c8e6..4c4a120e 100644 --- a/src/interlinearizer.web-view.tsx +++ b/src/interlinearizer.web-view.tsx @@ -1,41 +1,153 @@ import type { WebViewProps } from '@papi/core'; -import { useProjectData } from '@papi/frontend/react'; +import { + useLocalizedStrings, + useProjectData, + useProjectSetting, + useRecentScriptureRefs, +} from '@papi/frontend/react'; import { isPlatformError } from 'platform-bible-utils'; +import { useEffect, useMemo } from 'react'; +import { + BOOK_CHAPTER_CONTROL_STRING_KEYS, + BookChapterControl, + ScrollGroupSelector, + TabToolbar, +} from 'platform-bible-react'; +import { extractBookFromUsj } from 'parsers/papi/usjBookExtractor'; +import { tokenizeBook } from 'parsers/papi/bookTokenizer'; +import type { Book, Segment } from 'interlinearizer'; +import { logger } from '@papi/frontend'; + +const AVAILABLE_SCROLL_GROUPS = [undefined, 0, 1, 2, 3, 4]; /** - * Fetches and displays the USJ book data for the given project and scripture reference. Shows a - * loading indicator while data is in flight, an error message if the fetch fails or returns no - * data, and the raw JSON of the book otherwise. + * Renders the tokens of a single segment as inline chips. + * + * @param props - Component props + * @param props.segment - The segment whose tokens to render + * @param props.isActive - Whether this segment is the currently selected verse + * @param props.onClick - Callback invoked when the segment button is clicked + * @returns A button containing the segment's verse label and token chips + */ +function SegmentView({ + segment, + isActive, + onClick, +}: Readonly<{ + segment: Segment; + isActive?: boolean; + onClick?: () => void; +}>) { + return ( + + ); +} + +/** + * Fetches the USJ book for the given project, tokenizes it, and renders all segments in the current + * chapter. Shows loading / error states while data is in flight or unavailable. + * + * @param props - Component props + * @param props.projectId - PAPI project ID whose USJ book to fetch and tokenize + * @param props.scrRef - Current scripture reference shared via the scroll group + * @param props.setScrRef - Setter to update the scroll-group scripture reference when a segment is + * clicked + * @returns A column of {@link SegmentView} chips, or an appropriate loading / error message */ function ProjectBookFetcher({ projectId, scrRef, -}: { + setScrRef, +}: Readonly<{ projectId: string; scrRef: ReturnType[0]; -}) { + setScrRef: ReturnType[1]; +}>) { + const bookScrRef = useMemo( + () => ({ book: scrRef.book, chapterNum: 1, verseNum: 1 }), + [scrRef.book], + ); const [bookResult, , isLoading] = useProjectData('platformScripture.USJ_Book', projectId).BookUSJ( - scrRef, + bookScrRef, undefined, ); - let bookUsj: typeof bookResult | undefined; - let bookError: string | undefined; + const [writingSystem] = useProjectSetting(projectId, 'platform.languageTag', ''); + const [book, tokenizeError] = useMemo((): [ + Book | undefined, + { message: string; raw: unknown } | undefined, + ] => { + if (!bookResult || isPlatformError(bookResult)) return [undefined, undefined]; + try { + const ws = isPlatformError(writingSystem) ? 'und' : writingSystem || 'und'; + return [tokenizeBook(extractBookFromUsj(bookResult, ws)), undefined]; + } catch (err) { + return [undefined, { message: err instanceof Error ? err.message : String(err), raw: err }]; + } + }, [bookResult, writingSystem]); + + useEffect(() => { + if (tokenizeError) { + const ws = isPlatformError(writingSystem) ? 'und' : writingSystem || 'und'; + logger.error('Failed to parse/tokenize USJ book', tokenizeError.raw, { + message: tokenizeError.message, + writingSystem: ws, + projectId, + book: scrRef.book, + }); + } + }, [tokenizeError, writingSystem, projectId, scrRef.book]); + + const chapterSegments = useMemo( + () => + book?.segments.filter( + (seg) => seg.startRef.book === scrRef.book && seg.startRef.chapter === scrRef.chapterNum, + ) ?? [], + [book, scrRef.book, scrRef.chapterNum], + ); + + let bookError: string | undefined; if (isPlatformError(bookResult)) { bookError = bookResult.message; } else if (!isLoading && bookResult === undefined) { bookError = `No USJ book available for ${scrRef.book} in project ${projectId}`; - } else { - bookUsj = bookResult; } return ( - <> -

- {scrRef.book} · project {projectId} -

- +
{bookError && (

Error loading book

@@ -45,37 +157,103 @@ function ProjectBookFetcher({
)} - {!bookError && ( - 
-          {isLoading ? 'Loading…' : JSON.stringify(bookUsj, undefined, 2)}
-
+ {tokenizeError && ( +
+

Error processing book

+ 
+            {tokenizeError.message}
+
+
+ )} + + {!bookError && !tokenizeError && isLoading && ( +

Loading…

)} - + + {!bookError && !tokenizeError && !isLoading && chapterSegments.length === 0 && ( +

+ No verse data for {scrRef.book} {scrRef.chapterNum}. +

+ )} + + {!bookError && !tokenizeError && !isLoading && chapterSegments.length > 0 && ( +
+ {chapterSegments.map((seg) => ( + + setScrRef({ + book: seg.startRef.book, + chapterNum: seg.startRef.chapter, + verseNum: seg.startRef.verse, + }) + } + /> + ))} +
+ )} +
); } /** - * Root WebView component for the Interlinearizer. Reads the scroll-group scripture reference and - * delegates book fetching to {@link ProjectBookFetcher}. Shows a placeholder when no projectId is - * provided (i.e. the WebView was opened without a project). + * Root WebView component for the Interlinearizer. Renders a sticky reference picker at the top and + * delegates book fetching to {@link ProjectBookFetcher} in the scrollable content area below. + * + * @param props - WebView props injected by the PAPI host + * @param props.projectId - PAPI project ID passed from the host; undefined when the WebView is + * opened outside a project context + * @param props.useWebViewScrollGroupScrRef - Hook that exposes the shared scroll-group scripture + * reference and its setter + * @returns The full interlinearizer WebView layout */ globalThis.webViewComponent = function InterlinearizerWebView({ projectId, useWebViewScrollGroupScrRef, }: WebViewProps) { - const [scrRef] = useWebViewScrollGroupScrRef(); + const [scrRef, setScrRef, scrollGroupId, setScrollGroupId] = useWebViewScrollGroupScrRef(); + + const [localizedStrings] = useLocalizedStrings( + useMemo(() => [...BOOK_CHAPTER_CONTROL_STRING_KEYS], []), + ); + const { recentScriptureRefs: recentRefs, addRecentScriptureRef: onAddRecentRef } = + useRecentScriptureRefs(); return ( -
-

Interlinearizer

+
+ + } + endAreaChildren={ + + } + onSelectProjectMenuItem={() => {}} + onSelectViewInfoMenuItem={() => {}} + /> - {projectId ? ( - - ) : ( -

- Open this WebView from a Paratext project to load its source book. -

- )} +
+ {projectId ? ( + + ) : ( +

+ Open this WebView from a Paratext project to load its source book. +

+ )} +
); }; diff --git a/src/main.ts b/src/main.ts index 0b57c532..5793bcfc 100644 --- a/src/main.ts +++ b/src/main.ts @@ -17,6 +17,7 @@ const mainWebViewType = 'interlinearizer.mainWebView'; /** Options passed to `openWebView` when opening the Interlinearizer. */ export interface InterlinearizerOpenOptions extends OpenWebViewOptions { + /** Paratext project ID to load in the Interlinearizer WebView. */ projectId?: string; } @@ -28,15 +29,15 @@ const mainWebViewProvider: IWebViewProvider = { * * @param savedWebView - Platform-provided definition (webViewType, etc.). * @param openWebViewOptions - Options passed by the caller; may include a projectId to link. - * @returns WebView definition with title, content, and styles, or undefined. - * @throws {Error} When savedWebView.webViewType is not the Interlinearizer type. + * @returns WebView definition with title, content, and styles. + * @throws {TypeError} When savedWebView.webViewType is not the Interlinearizer type. */ async getWebView( savedWebView: SavedWebViewDefinition, openWebViewOptions?: InterlinearizerOpenOptions, ): Promise { if (savedWebView.webViewType !== mainWebViewType) { - throw new Error( + throw new TypeError( `${mainWebViewType} provider received request to provide a ${savedWebView.webViewType} WebView`, ); } @@ -60,7 +61,10 @@ const openWebViewsByProject = new Map(); /** * Opens the Interlinearizer WebView for the given project. If no projectId is provided, shows a * project picker dialog. Each project gets its own tab; reopening an already-open project brings - * that tab to front. Returns the WebView ID, or undefined if the user cancels. + * that tab to front. + * + * @param projectId - Project to open; if omitted a picker dialog is shown. + * @returns The WebView ID of the opened (or focused) tab, or `undefined` if the user cancels. */ async function openInterlinearizer(projectId?: string): Promise { const resolvedProjectId = @@ -82,6 +86,9 @@ async function openInterlinearizer(projectId?: string): Promise { if (!webViewId) return openInterlinearizer(); @@ -95,6 +102,7 @@ async function openInterlinearizerForWebView(webViewId?: string): Promise { logger.debug('Interlinearizer extension is activating!'); @@ -118,10 +126,14 @@ export async function activate(context: ExecutionActivationContext): Promise { + const surfaceText = match[0]; + const charStart = match.index; + const charEnd = charStart + surfaceText.length; + const type: TokenType = WORD_CONTAIN_RE.test(surfaceText) ? 'word' : 'punctuation'; + return { id: `${sid}:${charStart}`, surfaceText, writingSystem, type, charStart, charEnd }; + }); +} + +/** + * Tokenizes a {@link RawBook} into the interlinear model's `Book` (text layer only — no analysis). + * + * Each `RawVerse` becomes one `Segment`. The verse SID is parsed into `startRef` / `endRef` (both + * equal — verse-level granularity). The verse text is split into `Token`s using Unicode-aware + * word/punctuation splitting; character offsets are relative to `Segment.baselineText`. + * + * Invariant upheld for every token: `segment.baselineText.slice(token.charStart, token.charEnd) === + * token.surfaceText`. + * + * @param rawBook - Extracted book data from {@link extractBookFromUsj}. + * @returns A `Book` with one `Segment` per verse, each containing its ordered `Token`s. + * @throws {SyntaxError} If any `RawVerse.sid` cannot be parsed as a valid scripture reference. + */ +export function tokenizeBook(rawBook: RawBook): Book { + const segments: Segment[] = rawBook.verses.map(({ sid, text }) => { + const ref = parseSid(sid); + if (ref.book !== rawBook.bookCode) { + throw new SyntaxError(`Verse SID "${sid}" does not match book code "${rawBook.bookCode}"`); + } + return { + id: sid, + startRef: { ...ref }, + endRef: { ...ref }, + baselineText: text, + tokens: tokenizeVerse(text, sid, rawBook.writingSystem), + }; + }); + + return { + id: rawBook.bookCode, + bookRef: rawBook.bookCode, + textVersion: rawBook.contentHash, + segments, + }; +} diff --git a/src/parsers/papi/usjBookExtractor.ts b/src/parsers/papi/usjBookExtractor.ts new file mode 100644 index 00000000..66c564ee --- /dev/null +++ b/src/parsers/papi/usjBookExtractor.ts @@ -0,0 +1,287 @@ +/** @file Extracts {@link RawBook} from a papi USJ book response. */ + +/** Plain text of a single verse extracted from a USJ document, ready to be tokenized. */ +export interface RawVerse { + /** SID from the USJ verse marker, e.g. `"GEN 1:1"`. Parsed into `Segment.startRef` / `endRef`. */ + sid: string; + /** + * Accumulated plain-text content of the verse. Note and footnote content is excluded. Becomes + * `Segment.baselineText`; token `charStart` / `charEnd` are expressed relative to this string. + */ + text: string; +} + +/** + * Raw book data captured from a papi USJ response. Self-contained — everything the tokenizer needs + * to produce `Book → Segment → Token`. + */ +export interface RawBook { + /** 3-letter book code, e.g. `"GEN"`. */ + bookCode: string; + /** BCP 47 writing system tag for the baseline text, from `platform.languageTag`. */ + writingSystem: string; + /** FNV-1a hash of the serialized USJ content. Becomes `Book.textVersion`. */ + contentHash: string; + /** Verse entries in document order, one per USJ `verse` marker. */ + verses: RawVerse[]; +} + +// --------------------------------------------------------------------------- +// Minimal local types for USJ traversal. +// @eten-tech-foundation/scripture-utilities is not a direct dependency of this +// extension, so we define the subset we need here. +// --------------------------------------------------------------------------- + +/** A USJ content item: either a plain text string or a marker node. */ +type MarkerContent = string | UsjNode; + +/** A USJ marker node. Only the fields used during extraction are declared. */ +interface UsjNode { + /** Node type string (e.g. `"book"`, `"chapter"`, `"verse"`, `"para"`, `"note"`). */ + type: string; + /** USFM marker (e.g. `"p"`, `"s1"`, `"q"`). Present on `para` and `note` nodes. */ + marker?: string; + /** Chapter or verse number string. Present on `chapter` nodes. */ + number?: string; + /** 3-letter book code. Present on `book` nodes. */ + code?: string; + /** + * Verse or chapter SID. Present on `verse` nodes (e.g. `"GEN 1:1"`) and `chapter` nodes (e.g. + * `"GEN 1"`). + */ + sid?: string; + /** Child content items (strings or nested nodes). */ + content?: MarkerContent[]; +} + +/** Minimal shape of a USJ document as returned by the papi `platformScripture.USJ_Book` provider. */ +export interface UsjDocument { + content: MarkerContent[]; +} + +// --------------------------------------------------------------------------- +// Implementation +// --------------------------------------------------------------------------- + +/** + * Para markers whose content is not part of the verse baseline text (headings, titles, spacing, + * speaker IDs, acrostic headings, etc.). Verse-content para markers (p, m, pi, q*, etc.) are absent + * from this set and have their text accumulated as usual. + */ +const HEADING_PARA_MARKERS = new Set([ + // Major section headings and reference ranges + 'ms', + 'ms1', + 'ms2', + 'ms3', + 'mr', + // Section headings, reference ranges, and descriptive titles + 's', + 's1', + 's2', + 's3', + 's4', + 'sr', + 'r', + 'd', + // Speaker, acrostic heading, blank lines + 'sp', + 'qa', + 'b', + 'ib', + // Introduction headings + 'imt', + 'imt1', + 'imt2', + 'imt3', + 'imte', + 'imte1', + 'imte2', + 'is', + 'is1', + 'is2', +]); + +/** Mutable state threaded through the recursive USJ traversal. */ +interface TraversalState { + /** 3-letter book code captured from the `book` marker (e.g. `"GEN"`). */ + bookCode: string; + /** Verse SIDs seen so far; used to reject duplicates. */ + seenVerseIds: Set; + /** The verse currently being accumulated; `undefined` when outside a verse scope. */ + currentVerse: { sid: string; text: string } | undefined; + /** Completed verses in document order. */ + verses: RawVerse[]; +} + +/** + * Captures the book code from a `book` node, then recurses into its content. + * + * @param node - The `book` USJ node; `node.code` is the 3-letter book code. + * @param state - Shared traversal state updated in place. + */ +function handleBookNode(node: UsjNode, state: TraversalState): void { + if (node.code) state.bookCode = node.code; + if (node.content) traverse(node.content, state); +} + +/** + * Closes the current open verse (if any) when a `chapter` node is encountered, then recurses into + * the chapter's content to pick up verses inside it. + * + * @param node - The `chapter` USJ node. + * @param state - Shared traversal state updated in place. + */ +function handleChapterNode(node: UsjNode, state: TraversalState): void { + if (state.currentVerse !== undefined) { + state.currentVerse.text = state.currentVerse.text.trimEnd(); + state.verses.push(state.currentVerse); + state.currentVerse = undefined; + } + if (node.content) traverse(node.content, state); +} + +/** + * Closes the previous open verse (if any) and opens a new one for a `verse` node. + * + * @param node - The `verse` USJ node; must carry a `sid` attribute (e.g. `"GEN 1:1"`). + * @param state - Shared traversal state updated in place. + * @throws {SyntaxError} If the `verse` node is missing its required `sid` attribute. + */ +function handleVerseNode(node: UsjNode, state: TraversalState): void { + if (state.currentVerse !== undefined) { + state.currentVerse.text = state.currentVerse.text.trimEnd(); + state.verses.push(state.currentVerse); + } + if (!node.sid) throw new SyntaxError('Invalid USJ: verse marker missing required sid attribute'); + if (state.seenVerseIds.has(node.sid)) + throw new SyntaxError(`Invalid USJ: duplicate verse SID "${node.sid}"`); + state.seenVerseIds.add(node.sid); + state.currentVerse = { sid: node.sid, text: '' }; + if (node.content) traverse(node.content, state); +} + +/** + * Recurses into a `para` node's content, appending a space between adjacent para nodes when needed. + * Heading-class paragraphs (see {@link HEADING_PARA_MARKERS}) are skipped entirely so their text is + * not included in the verse baseline. + * + * @param node - The `para` USJ node; `node.marker` determines whether to skip or recurse. + * @param state - Shared traversal state updated in place. + */ +function handleParaNode(node: UsjNode, state: TraversalState): void { + if (node.marker && HEADING_PARA_MARKERS.has(node.marker)) return; + if ( + state.currentVerse !== undefined && + state.currentVerse.text.length > 0 && + !state.currentVerse.text.endsWith(' ') + ) + state.currentVerse.text += ' '; + if (node.content) traverse(node.content, state); +} + +/** Dispatch table mapping USJ node `type` strings to their traversal handlers. */ +const NODE_HANDLERS: Partial void>> = { + book: handleBookNode, + chapter: handleChapterNode, + verse: handleVerseNode, + note: () => {}, // skip note/footnote content — not part of the baseline text + para: handleParaNode, +}; + +/** + * Recursively walks a USJ content array, accumulating verse text into `state`. + * + * @param nodes - Content items to walk (`string` or {@link UsjNode}). + * @param state - Shared mutable state updated in place during traversal. + */ +function traverse(nodes: MarkerContent[], state: TraversalState): void { + nodes.forEach((node) => { + if (typeof node === 'string') { + if (state.currentVerse !== undefined) state.currentVerse.text += node; + return; + } + const handler = Object.hasOwn(NODE_HANDLERS, node.type) ? NODE_HANDLERS[node.type] : undefined; + if (handler) handler(node, state); + else if (node.content) traverse(node.content, state); + }); +} + +/** + * Deterministic JSON serialization with keys sorted by UTF-16 code-unit order. + * + * Produces the same output regardless of engine locale, making the result safe to feed into a hash + * function. Arrays preserve their original order; only object keys are sorted. + * + * Intended for plain JSON-shaped structures only; does not special-case Date, Map, Set, or RegExp. + * + * @param value - Any JSON-serializable value. + * @returns A stable JSON string with object keys in UTF-16 code-unit order. + */ +function stableStringify(value: unknown): string { + if (value === undefined) return 'null'; + if (!(value instanceof Object)) return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`; + const sorted = Object.entries(value) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => +(a > b) - +(a < b)) + .map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`); + return `{${sorted.join(',')}}`; +} + +/** + * FNV-1a 32-bit hash — sufficient for one-way internal content versioning. + * + * @param s - String to hash. + * @returns Lowercase hex string of the unsigned 32-bit FNV-1a digest. + */ +function fnv1a32(s: string): string { + let h = 2166136261; + // eslint-disable-next-line no-restricted-syntax -- iterating over string, not array + for (const char of s) { + /* v8 ignore next 2 -- codePointAt(0) on a spread char is always defined */ + // eslint-disable-next-line no-bitwise + h = Math.imul(h ^ (char.codePointAt(0) ?? 0), 16777619); + } + // eslint-disable-next-line no-bitwise + return (h >>> 0).toString(16).padStart(8, '0'); +} + +/** + * Extracts a {@link RawBook} from a papi USJ book response. + * + * Each `verse` marker in the USJ document becomes one {@link RawVerse}. Text strings within the + * verse scope are accumulated into `RawVerse.text`; `note` nodes are skipped entirely. Verse + * markers with no following text produce an empty `RawVerse` (`text: ""`). + * + * @param usj - USJ document returned by `useProjectData('platformScripture.USJ_Book', ...)`. + * @param writingSystem - BCP 47 tag for the baseline, from `platform.languageTag`. + * @returns A `RawBook` with `bookCode`, `writingSystem`, `contentHash`, and `verses` populated. + * @throws {SyntaxError} If no `book` marker with a `code` attribute is found in the document. + */ +export function extractBookFromUsj(usj: UsjDocument, writingSystem: string): RawBook { + const contentHash = fnv1a32(stableStringify(usj.content)); + const state: TraversalState = { + bookCode: '', + seenVerseIds: new Set(), + currentVerse: undefined, + verses: [], + }; + + traverse(usj.content, state); + + if (state.currentVerse !== undefined) { + state.currentVerse.text = state.currentVerse.text.trimEnd(); + state.verses.push(state.currentVerse); + } + + if (!state.bookCode) + throw new SyntaxError('Invalid USJ: no book marker with a code attribute found'); + + return { + bookCode: state.bookCode, + writingSystem, + contentHash, + verses: state.verses, + }; +} diff --git a/src/parsers/interlinearXmlParser.ts b/src/parsers/pt9/interlinearXmlParser.ts similarity index 82% rename from src/parsers/interlinearXmlParser.ts rename to src/parsers/pt9/interlinearXmlParser.ts index 63f0dc9d..765b6287 100644 --- a/src/parsers/interlinearXmlParser.ts +++ b/src/parsers/pt9/interlinearXmlParser.ts @@ -142,7 +142,7 @@ interface ParsedInterlinearXml { * * @param clusterElement - Parsed Cluster from fast-xml-parser (may have Lexeme array or none). * @returns Array of LexemeData with LexemeId (from Id) and SenseId (from GlossId, or ''). - * @throws {Error} If any Lexeme element is missing the required Id attribute. + * @throws {SyntaxError} If any Lexeme element is missing the required Id attribute. */ function extractLexemesFromCluster(clusterElement: ParsedCluster): LexemeData[] { const elements = clusterElement.Lexeme ?? []; @@ -150,12 +150,28 @@ function extractLexemesFromCluster(clusterElement: ParsedCluster): LexemeData[] return elements.map((el) => { const lexemeId = el['@_Id']; if (!lexemeId) { - throw new Error('Invalid XML: Lexeme missing required Id attribute'); + throw new SyntaxError('Invalid XML: Lexeme missing required Id attribute'); } return { LexemeId: lexemeId, SenseId: el['@_GlossId'] ?? '' }; }); } +/** + * Parses a string to a non-negative integer, returning `undefined` for empty, non-integer, or + * negative values. Used to validate XML attribute strings before converting them to numeric + * ranges. + * + * @param raw - The string to parse (e.g. an XML attribute value). + * @returns The parsed non-negative integer, or `undefined` if the input is empty, non-integer, or + * negative. + */ +const parseStrictNumber = (raw: string): number | undefined => { + if (raw === undefined || raw.trim() === '') return undefined; + if (!/^\d+$/.test(raw.trim())) return undefined; + const n = Number.parseInt(raw, 10); + return n >= 0 ? n : undefined; +}; + /** * Maps a parsed VerseData's Punctuation array to {@link PunctuationData} array. * @@ -174,9 +190,9 @@ function extractPunctuationsFromVerse(verseDataElement: ParsedVerseData): Punctu return elements.flatMap((el) => { const rangeElement = el.Range; if (!rangeElement) return []; - const index = Number(rangeElement['@_Index']); - const length = Number(rangeElement['@_Length']); - if (!Number.isFinite(index) || !Number.isFinite(length)) return []; + const index = parseStrictNumber(rangeElement['@_Index']); + const length = parseStrictNumber(rangeElement['@_Length']); + if (index === undefined || length === undefined) return []; return [ { TextRange: { Index: index, Length: length }, @@ -193,7 +209,8 @@ function extractPunctuationsFromVerse(verseDataElement: ParsedVerseData): Punctu * @param verseDataElement - Parsed VerseData from fast-xml-parser (may have Cluster array or none). * @returns Array of ClusterData: TextRange from Cluster's Range, Lexemes from Lexeme children, * LexemesId (slash-joined), Id (LexemesId/Index-Length or Index-Length when no lexemes). - * @throws {Error} If a Cluster is missing its Range element or Range is missing Index or Length. + * @throws {SyntaxError} If a Cluster is missing its Range element or Range is missing Index or + * Length. */ function extractClustersFromVerse(verseDataElement: ParsedVerseData): ClusterData[] { const clusterElements = verseDataElement.Cluster ?? []; @@ -201,13 +218,15 @@ function extractClustersFromVerse(verseDataElement: ParsedVerseData): ClusterDat return clusterElements.map((el) => { const rangeElement = el.Range; if (!rangeElement) { - throw new Error('Invalid XML: Cluster missing required Range element'); + throw new SyntaxError('Invalid XML: Cluster missing required Range element'); } - const index = Number(rangeElement['@_Index']); - const length = Number(rangeElement['@_Length']); - if (!Number.isFinite(index) || !Number.isFinite(length)) { - throw new Error('Invalid XML: Range missing required Index or Length attributes'); + const index = parseStrictNumber(rangeElement['@_Index']); + const length = parseStrictNumber(rangeElement['@_Length']); + if (index === undefined || length === undefined) { + throw new SyntaxError( + 'Invalid XML: Range missing or invalid Index/Length attributes (must be non-negative integers)', + ); } const textRange: StringRange = { Index: index, Length: length }; @@ -272,39 +291,42 @@ export class InterlinearXmlParser { * entries. * @returns Parsed interlinear data: ScrTextName, GlossLanguage, BookId, and Verses (record of * verse key to {@link VerseData} with Hash, Clusters, Punctuations). - * @throws {Error} If the root element, required attributes (GlossLanguage, BookId), required - * structure (Verses, Cluster Range, Lexeme Id), or duplicate verse reference is present. + * @throws {SyntaxError} If the root element, required attributes (GlossLanguage, BookId), or + * required structure (Verses, Cluster Range, Lexeme Id) is missing. + * @throws {SyntaxError} If a verse reference appears more than once. */ parse(xml: string): InterlinearData { const parsed: ParsedInterlinearXml = this.parser.parse(xml); const root = parsed.InterlinearData; if (!root) { - throw new Error('Invalid XML: Missing InterlinearData root element'); + throw new SyntaxError('Invalid XML: Missing InterlinearData root element'); } const scrTextName = root['@_ScrTextName'] ?? ''; const glossLanguage = root['@_GlossLanguage'] ?? ''; const bookId = root['@_BookId'] ?? ''; if (!glossLanguage || !bookId) { - throw new Error('Invalid XML: Missing required attributes GlossLanguage or BookId'); + throw new SyntaxError('Invalid XML: Missing required attributes GlossLanguage or BookId'); } const versesElement = root.Verses; if (!versesElement) { - throw new Error('Invalid XML: Missing Verses element'); + throw new SyntaxError('Invalid XML: Missing Verses element'); } const items = versesElement.item ?? []; + const seen = new Set(); const verses = items.reduce>((acc, item) => { const verseKey = item.string; if (!verseKey) return acc; - if (verseKey in acc) { - throw new Error( + if (seen.has(verseKey)) { + throw new SyntaxError( `Invalid XML: Duplicate verse reference "${verseKey}". At most one VerseData per reference is allowed.`, ); } + seen.add(verseKey); const verseDataElement = item.VerseData; if (!verseDataElement) { diff --git a/src/parsers/pt9-xml.md b/src/parsers/pt9/pt9-xml.md similarity index 97% rename from src/parsers/pt9-xml.md rename to src/parsers/pt9/pt9-xml.md index 56d0baac..b570279e 100644 --- a/src/parsers/pt9-xml.md +++ b/src/parsers/pt9/pt9-xml.md @@ -1,6 +1,6 @@ # Paratext 9 XML schema -The extension reads PT9 interlinear data from XML files (e.g. `Interlinear__.xml` in project data). The parser in `src/parsers/interlinearXmlParser.ts` expects the following structure. Sample files live in `test-data/` (e.g. `Interlinear_en_MAT.xml`). +The extension reads PT9 interlinear data from XML files (e.g. `Interlinear__.xml` in project data). The parser in `src/parsers/pt9/interlinearXmlParser.ts` expects the following structure. Sample files live in `test-data/` (e.g. `Interlinear_en_MAT.xml`). ## Document structure diff --git a/src/types/interlinearizer.d.ts b/src/types/interlinearizer.d.ts index d4ecd542..009c1668 100644 --- a/src/types/interlinearizer.d.ts +++ b/src/types/interlinearizer.d.ts @@ -71,7 +71,7 @@ declare module 'interlinearizer' { export type TokenType = 'word' | 'punctuation'; /** - * How an analysis was produced. + * Confidence level of an analysis. * * - `high` — human-created or human-confirmed * - `medium` — tool-assisted, reasonably confident @@ -105,8 +105,11 @@ declare module 'interlinearizer' { * text. When `charIndex` is absent the reference is verse-level only. */ export interface ScriptureRef { + /** 3-letter SIL book code (e.g. `"GEN"`). */ book: string; + /** 1-based chapter number. */ chapter: number; + /** 1-based verse number. */ verse: number; /** Zero-based character offset within the verse's baseline text. */ charIndex?: number; @@ -232,6 +235,7 @@ declare module 'interlinearizer' { * `InterlinearText`. `Alignment` records become `AlignmentLink`s. */ export interface InterlinearAlignment { + /** Unique identifier for this alignment pair. */ id: string; /** @@ -280,6 +284,7 @@ declare module 'interlinearizer' { * `senseIds`. Analysis is typically in a single language. */ export interface InterlinearText { + /** Unique identifier for this interlinear text. */ id: string; /** Writing system of the baseline text. */ @@ -318,6 +323,7 @@ declare module 'interlinearizer' { * from token checksums at import time. */ export interface Book { + /** Unique identifier for this book; typically equal to `bookRef`. */ id: string; /** Book identifier (e.g. `"GEN"`, `"MAT"`). */ @@ -519,6 +525,10 @@ declare module 'interlinearizer' { * synthesized. */ export interface SegmentAnalysis { + /** + * Unique within the owning `TextAnalysis` — used as a stable reference for this analysis + * record. + */ id: string; /** @@ -749,6 +759,7 @@ declare module 'interlinearizer' { * share the same gloss / sense. */ export type Phrase = { + /** Unique within the owning `TextAnalysis` — used as a stable reference for this phrase record. */ id: string; /** Ordered `Token.id` values that compose this phrase. */ @@ -813,6 +824,7 @@ declare module 'interlinearizer' { * Eflomal-generated alignments leave `originNum` and `statusNum` unset (default 0, CREATED). */ export interface AlignmentLink { + /** Unique within the owning `InterlinearAlignment` — stable reference for this link. */ id: string; /** Source-side endpoints (one or more tokens / morphemes). */ @@ -821,6 +833,7 @@ declare module 'interlinearizer' { /** Target-side endpoints (one or more tokens / morphemes). */ targetEndpoints: AlignmentEndpoint[]; + /** Review status of this alignment link. */ status: AssignmentStatus; /** How the alignment was created (manual, automatic tool, etc.). */