diff --git a/src/internal/script.ts b/src/internal/script.ts index 07639f4..6e08051 100644 --- a/src/internal/script.ts +++ b/src/internal/script.ts @@ -46,23 +46,65 @@ export function collectScriptRanges( jsx = false, jsxAtLineStart = false, ) { - const lexical = collectScriptLexicalRanges(code, jsx, jsxAtLineStart) - const markup = jsx ? collectJsxRanges(code, jsxAtLineStart) : [] - return collectPatternRanges(code, semanticPatterns, [...lexical, ...markup]) + const jsxText = jsx ? new Uint8Array(code.length) : undefined + const initial: Array = [] + collectScriptInitialRanges( + code, + jsx, + jsxAtLineStart, + jsxText, + initial, + ) + const ranges = collectPatternRanges(code, semanticPatterns, initial) + return jsxText + ? ranges.filter((range) => !jsxText[range.start]) + : ranges } -function collectScriptLexicalRanges( +function collectScriptInitialRanges( code: string, jsx: boolean, jsxAtLineStart: boolean, + jsxText: Uint8Array | undefined, + ranges: Array, + index = 0, + limit = code.length, + tagBody = false, ) { - const ranges: Array = [] - let index = 0 + const expressions: Array = [] + let jsxDepth = 0 - while (index < code.length) { + while (index < limit) { + const expression = expressions.at(-1) + const inJsxText = jsxDepth > (expression || 0) const character = code[index] const next = code[index + 1] + if (inJsxText) { + if (character === '{') { + expressions.push(jsxDepth) + index++ + continue + } + if (character !== '<') { + const start = index + while (index < code.length && code[index] !== '<' && code[index] !== '{') { + index++ + } + jsxText!.fill(1, start, index) + continue + } + } + + if ( + tagBody && + !expressions.length && + !jsxDepth && + character === '>' + ) { + return index + } + if (character === '/' && next === '/') { const end = findLineEnd(code, index + 2) ranges.push({ start: index, end, className: 'comment' }) @@ -79,7 +121,12 @@ function collectScriptLexicalRanges( } if (character === "'" || character === '"') { - const end = findQuotedEnd(code, index, character) + const end = findQuotedEnd( + code, + index, + character, + tagBody && !expressions.length, + ) ranges.push({ start: index, end, className: 'string' }) index = end continue @@ -103,10 +150,90 @@ function collectScriptLexicalRanges( } } - index++ + if (!jsx) { + index++ + continue + } + + if ( + !inJsxText && + character === '{' && + (expressions.length || tagBody) + ) { + expressions.push(expressions.length ? expression! : jsxDepth) + index++ + continue + } + if (!inJsxText && expressions.length && character === '}') { + expressions.pop() + index++ + continue + } + + if (character !== '<') { + index++ + continue + } + + if (code.startsWith('', index)) { + jsxDepth-- + index += 3 + continue + } + if ( + code.startsWith('<>', index) && + (inJsxText || isJsxStart(code, index, jsxAtLineStart)) + ) { + jsxDepth++ + index += 2 + continue + } + + const closing = code[index + 1] === '/' + const nameStart = index + (closing ? 2 : 1) + const nameMatch = /^[A-Za-z][\w:.-]*/.exec(code.slice(nameStart)) + if ( + !nameMatch || + (!closing && + !inJsxText && + !isJsxStart(code, index, jsxAtLineStart)) + ) { + index++ + continue + } + + const end = collectScriptInitialRanges( + code, + true, + jsxAtLineStart, + jsxText, + ranges, + nameStart + nameMatch[0].length, + limit, + true, + ) + if (end < 0 || isTypeParameter(code, index, end, closing)) { + index++ + continue + } + + ranges.push({ + start: nameStart, + end: nameStart + nameMatch[0].length, + className: 'tag', + }) + + if (!closing) { + const attributeStart = nameStart + nameMatch[0].length + collectJsxAttributes(code, attributeStart, end, ranges) + } + const selfClosing = /\/\s*$/.test(code.slice(nameStart, end)) + if (closing) jsxDepth-- + else if (!selfClosing) jsxDepth++ + index = end + 1 } - return ranges + return -1 } function collectTemplateRanges( @@ -208,86 +335,6 @@ function skipTemplate(code: string, start: number) { return code.length } -function collectJsxRanges(code: string, jsxAtLineStart: boolean) { - const ranges: Array = [] - let index = 0 - let jsxDepth = 0 - let expressionDepth = 0 - - while (index < code.length) { - if (jsxDepth && (code[index] === '"' || code[index] === "'" || code[index] === '`')) { - index = code[index] === '`' - ? skipTemplate(code, index) - : findQuotedEnd(code, index, code[index]) - continue - } - - if (jsxDepth && code[index] === '{') { - expressionDepth++ - index++ - continue - } - if (jsxDepth && code[index] === '}') { - expressionDepth = Math.max(0, expressionDepth - 1) - index++ - continue - } - - if (code[index] !== '<') { - index++ - continue - } - - if (code.startsWith('', index)) { - jsxDepth = Math.max(0, jsxDepth - 1) - index += 3 - continue - } - if ( - code.startsWith('<>', index) && - (jsxDepth > 0 || isJsxStart(code, index, jsxAtLineStart)) - ) { - jsxDepth++ - index += 2 - continue - } - - const closing = code[index + 1] === '/' - const nameStart = index + (closing ? 2 : 1) - const nameMatch = /^[A-Za-z][\w:.-]*/.exec(code.slice(nameStart)) - const inJsxText = jsxDepth > 0 && expressionDepth === 0 - if ( - !nameMatch || - (!closing && - !inJsxText && - !isJsxStart(code, index, jsxAtLineStart)) - ) { - index++ - continue - } - - const end = findTagEnd(code, nameStart + nameMatch[0].length) - if (end < 0 || isTypeParameter(code, index, end, closing)) { - index++ - continue - } - - ranges.push({ - start: nameStart, - end: nameStart + nameMatch[0].length, - className: 'tag', - }) - - if (!closing) collectJsxAttributes(code, nameStart + nameMatch[0].length, end, ranges) - const selfClosing = /\/\s*$/.test(code.slice(nameStart, end)) - if (closing) jsxDepth = Math.max(0, jsxDepth - 1) - else if (!selfClosing) jsxDepth++ - index = end + 1 - } - - return ranges -} - function collectJsxAttributes( code: string, start: number, @@ -298,8 +345,7 @@ function collectJsxAttributes( const regex = /\s([:@A-Za-z_$][\w$:.-]*)(?=\s*(?:=|\/?>))/g let match: RegExpExecArray | null while ((match = regex.exec(source))) { - const offset = match[0].indexOf(match[1]) - const attributeStart = start + match.index + offset + const attributeStart = start + match.index + 1 ranges.push({ start: attributeStart, end: attributeStart + match[1].length, @@ -366,30 +412,17 @@ function findRegexEnd(code: string, start: number) { return start + 1 } -function findTagEnd(code: string, start: number) { - let braceDepth = 0 - let index = start - while (index < code.length) { - const character = code[index] - if (character === "'" || character === '"') { - index = findQuotedEnd(code, index, character) - continue - } - if (character === '{') braceDepth++ - else if (character === '}') braceDepth = Math.max(0, braceDepth - 1) - else if (character === '>' && braceDepth === 0) return index - else if (character === '\n' && braceDepth === 0) return -1 - index++ - } - return -1 -} - -function findQuotedEnd(code: string, start: number, quote: string) { +function findQuotedEnd( + code: string, + start: number, + quote: string, + multiline = false, +) { let index = start + 1 while (index < code.length) { if (code[index] === '\\') index += 2 else if (code[index] === quote) return index + 1 - else if (code[index] === '\n' && quote !== '`') return index + else if (!multiline && code[index] === '\n') return index else index++ } return code.length diff --git a/test/regressions.test.ts b/test/regressions.test.ts index 2b2586e..6671856 100644 --- a/test/regressions.test.ts +++ b/test/regressions.test.ts @@ -5,6 +5,7 @@ import { dockerfile } from '../src/languages/dockerfile' import { html } from '../src/languages/html' import { http } from '../src/languages/http' import { js } from '../src/languages/js' +import { jsx } from '../src/languages/jsx' import { markdown } from '../src/languages/markdown' import { python } from '../src/languages/python' import { shell } from '../src/languages/shell' @@ -21,6 +22,7 @@ const highlighter = createHighlighter({ html, http, js, + jsx, markdown, python, shell, @@ -46,11 +48,16 @@ describe('script context', () => { 'const pair = (key: Key, value: Value) => [key, value]', { lang: 'tsx' }, ) + const multiline = highlighter.tokenize( + 'const pair = (key: Key, value: Value) => [key, value]', + { lang: 'tsx' }, + ) expect(classesFor(generic, 'T')).toContain('type') expect(classesFor(generic, 'T')).not.toContain('tag') expect(classesFor(element, 'Component')).toContain('tag') expect(classesFor(multiple, 'Key')).not.toContain('tag') + expect(classesFor(multiline, 'Key')).not.toContain('tag') expect(reconstruct(generic)).toBe(generic.code) expect(reconstruct(element)).toBe(element.code) }) @@ -85,6 +92,132 @@ describe('script context', () => { ).toBe(false) }) + it('keeps apostrophes in JSX text from swallowing closing tags', () => { + const cases = [ + { + code: `We'll never share your email.`, + textEnd: '', + }, + { + code: `Don't {say('hi')}`, + textEnd: '{', + }, + ] + + for (const lang of ['jsx', 'tsx']) { + for (const { code, textEnd } of cases) { + const result = highlighter.tokenize(code, { lang }) + const textStart = code.indexOf('>') + 1 + + expect( + classesInRange(result, textStart, code.indexOf(textEnd)), + lang, + ).toEqual([]) + expect(exactClassesFor(result, 'Description'), lang).toEqual([ + 'tag', + 'tag', + ]) + if (code.includes("'hi'")) { + expect(exactClassesFor(result, "'hi'"), lang).toEqual(['string']) + } + expect(reconstruct(result), lang).toBe(code) + } + } + }) + + it('classifies multiline JSX tags and attributes', () => { + const code = ` null} +/>` + + for (const lang of ['jsx', 'tsx']) { + const result = highlighter.tokenize(code, { lang }) + + expect(exactClassesFor(result, 'AriaLink'), lang).toEqual(['tag']) + expect(exactClassesFor(result, 'href'), lang).toContain('attr') + expect(exactClassesFor(result, 'aria-label'), lang).toEqual(['attr']) + expect(exactClassesFor(result, '"Docs\n reference"'), lang).toEqual([ + 'string', + ]) + expect(exactClassesFor(result, 'render'), lang).toEqual(['attr']) + expect(classesFor(result, 'props'), lang).not.toContain('attr') + expect(classesFor(result, 'null'), lang).toContain('literal') + expect(reconstruct(result), lang).toBe(code) + } + }) + + it('classifies JSX nested in attribute expressions', () => { + const code = `can't} />` + + for (const lang of ['jsx', 'tsx']) { + const result = highlighter.tokenize(code, { lang }) + const textStart = code.indexOf("can't") + + expect(exactClassesFor(result, 'Wrapper'), lang).toEqual(['tag']) + expect(exactClassesFor(result, 'child'), lang).toEqual(['attr']) + expect(exactClassesFor(result, 'Label'), lang).toEqual(['tag', 'tag']) + expect(classesInRange(result, textStart, textStart + 5), lang).toEqual([]) + expect(reconstruct(result), lang).toBe(code) + } + }) + + it('leaves JSX text prose unclassified', () => { + const cases = [ + 'How do I get started?', + 'Option 1', + ] + + for (const lang of ['jsx', 'tsx']) { + for (const code of cases) { + const result = highlighter.tokenize(code, { lang }) + const textStart = code.indexOf('>') + 1 + const textEnd = code.lastIndexOf('<') + + expect(classesInRange(result, textStart, textEnd), `${lang}: ${code}`).toEqual( + [], + ) + expect(reconstruct(result), lang).toBe(code) + } + } + }) + + it('separates nested JSX text from its surrounding expression', () => { + const code = + 'return
{ready ? Get 1 item : null}
' + const result = highlighter.tokenize(code, { lang: 'tsx' }) + const textStart = code.indexOf('Get') + const textEnd = textStart + 'Get 1 item'.length + + expect(classesInRange(result, textStart, textEnd)).toEqual([]) + expect(exactClassesFor(result, 'strong')).toEqual(['tag', 'tag']) + expect(classesFor(result, 'null')).toContain('literal') + expect(reconstruct(result)).toBe(code) + }) + + it('tracks fragments and lexical syntax around JSX boundaries', () => { + const fragment = + 'return <>Text 2{ready ? "}" : /* > */ null}' + const expression = highlighter.tokenize(fragment, { lang: 'tsx' }) + const textStart = fragment.indexOf('Text') + const textEnd = textStart + 'Text 2'.length + const script = highlighter.tokenize( + 'const pattern = /(?:)/\nconst ready = true', + { lang: 'tsx' }, + ) + + expect(classesInRange(expression, textStart, textEnd)).toEqual([]) + expect(exactClassesFor(expression, 'span')).toEqual(['tag', 'tag']) + expect(classesFor(expression, 'null')).toContain('literal') + expect(classesFor(script, '/(?:)/')).toContain('literal') + expect(classesFor(script, 'true')).toContain('literal') + expect(exactClassesFor(script, 'Component')).not.toContain('tag') + expect(reconstruct(expression)).toBe(fragment) + expect(reconstruct(script)).toBe(script.code) + }) + it('tokenizes template interpolations recursively', () => { const result = highlighter.tokenize( 'const text = `hello ${user.name.toUpperCase()} ${`nested ${count}`}`', @@ -279,6 +412,34 @@ function classesFor( .map((token) => token.className) } +function exactClassesFor( + result: ReturnType, + value: string, +) { + return result.tokens + .filter((token) => token.value === value) + .map((token) => token.className) +} + +function classesInRange( + result: ReturnType, + start: number, + end: number, +) { + const classes: Array = [] + let offset = 0 + + for (const token of result.tokens) { + const tokenEnd = offset + token.value.length + if (token.className && tokenEnd > start && offset < end) { + classes.push(token.className) + } + offset = tokenEnd + } + + return classes +} + function reconstruct(result: ReturnType) { return result.tokens.map((token) => token.value).join('') }