diff --git a/src/chat_sdk/shared/base_format_converter.py b/src/chat_sdk/shared/base_format_converter.py index 0f6f7e4..2ddc5aa 100644 --- a/src/chat_sdk/shared/base_format_converter.py +++ b/src/chat_sdk/shared/base_format_converter.py @@ -100,7 +100,11 @@ def _render_list( ) -> str: """Render a list node with proper indentation. - Handles ordered and unordered lists and recurses into nested lists. + Handles ordered and unordered lists, recurses into nested lists, + and emits GFM task-list checkbox markers (``[ ]`` / ``[x]``) for + items carrying a ``checked`` attribute. Most chat platforms + either render the marker natively (GitHub, Linear) or display it + as literal text -- either way, dropping it would lose user data. """ indent = " " * depth start = node.get("start", 1) @@ -109,19 +113,37 @@ def _render_list( for i, item in enumerate(get_node_children(node)): prefix = f"{start + i}." if ordered else unordered_bullet + # GFM task-list marker, only on unordered items. + checked = item.get("checked") if not ordered else None + task_marker = "" + if checked is True: + task_marker = "[x] " + elif checked is False: + task_marker = "[ ] " is_first_content = True for child in get_node_children(item): if child.get("type") == "list": + if is_first_content: + # No text child before this nested list -- emit + # the parent prefix on its own line BEFORE the + # nested list, or the checkbox/parent ordering + # ends up reversed (PR #101 review finding). + lines.append(f"{indent}{prefix} {task_marker}".rstrip()) + is_first_content = False lines.append(self._render_list(child, depth + 1, node_converter, unordered_bullet)) continue text = node_converter(child) if not text.strip(): continue if is_first_content: - lines.append(f"{indent}{prefix} {text}") + lines.append(f"{indent}{prefix} {task_marker}{text}") is_first_content = False else: lines.append(f"{indent} {text}") + # Item with zero rendered children -- still emit a line so the + # listItem (and any task marker) isn't lost. + if is_first_content: + lines.append(f"{indent}{prefix} {task_marker}".rstrip()) return "\n".join(lines) diff --git a/src/chat_sdk/shared/markdown_parser.py b/src/chat_sdk/shared/markdown_parser.py index 78a104e..c933992 100644 --- a/src/chat_sdk/shared/markdown_parser.py +++ b/src/chat_sdk/shared/markdown_parser.py @@ -104,9 +104,16 @@ def make_list(children: list[Content], *, ordered: bool = False, start: int = 1) return node -def make_list_item(children: list[Content]) -> Content: - """Create a list item node.""" - return {"type": "listItem", "children": children} +def make_list_item(children: list[Content], *, checked: bool | None = None) -> Content: + """Create a list item node. + + *checked* follows the mdast GFM task-list extension: ``True`` for + ``- [x]``, ``False`` for ``- [ ]``, ``None`` for a regular list item. + """ + node: Content = {"type": "listItem", "children": children} + if checked is not None: + node["checked"] = checked + return node def make_thematic_break() -> Content: @@ -162,51 +169,287 @@ def get_node_value(node: Content) -> str: # Inline parser # --------------------------------------------------------------------------- -# Regex patterns for inline elements, ordered by priority. -# Each pattern captures a full inline construct. +# ASCII-punctuation characters that may be backslash-escaped per +# CommonMark. A preceding `\` makes the next character literal -- not a +# delimiter -- which `_protect_escapes` consumes into a sentinel-prefixed +# pair before the inline regexes run. +_ESCAPABLE_PUNCT = set("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") + +# Sentinel character used by `_protect_escapes` to mark every ``\X`` +# escape sequence as ``X``. U+FDD0 is one of the 32 BMP +# "noncharacters" permanently reserved by the Unicode standard for +# process-internal application use -- by definition, no real text +# interchange uses these codepoints. `_protect_escapes` also strips +# any pre-existing occurrence from the input as a defense-in-depth +# measure, so the sentinel-based round trip is robust even when an +# adversarial caller smuggles the codepoint in. +_ESCAPE_SENTINEL = "﷐" + + +def _protect_escapes(text: str) -> str: + """Replace every ``\\X`` escape pair with ``X`` in *text*. + + Used as a pre-pass before the inline regex matching. Each escaped + delimiter character is marked so the regex lookbehinds can skip it + via ``(?)``. Unlike the previous ``(? str: + """Inverse of :func:`_protect_escapes` for text-leaf emission. + + Replaces each ``X`` pair with literal ``X``. The + backslash is dropped because the escape sequence resolved to a + literal character at the AST text leaf. + + A *dangling* sentinel (one with no following char, which arises when + the escaped character was consumed as a delimiter -- e.g. a code + span ending in ``\\``, where the backtick that followed the sentinel + became the closing delimiter) resolves to a literal backslash. This + must never emit the raw sentinel codepoint, which would leak U+FDD0 + into user-visible output. + """ + if _ESCAPE_SENTINEL not in text: + return text + out: list[str] = [] + i = 0 + while i < len(text): + if text[i] == _ESCAPE_SENTINEL: + if i + 1 < len(text): + out.append(text[i + 1]) + i += 2 + else: + # Dangling sentinel: the escaped char was consumed + # elsewhere; keep the backslash, never the sentinel. + out.append("\\") + i += 1 + continue + out.append(text[i]) + i += 1 + return "".join(out) + + +def _restore_escapes_as_literal_pair(text: str) -> str: + """Inverse for inline-code emission: ``X`` -> ``\\X``. + + Per CommonMark, backslashes inside inline code spans are preserved + literally -- ``\\*`` inside a code span stays as ``\\*``, not ``*``. + + A dangling sentinel (e.g. from a code span ending in ``\\`` where the + closing backtick was the sentinel's paired char) resolves to a bare + backslash, matching CommonMark's "backslash is literal in code" + rule. Never emits the raw sentinel codepoint. + """ + if _ESCAPE_SENTINEL not in text: + return text + out: list[str] = [] + i = 0 + while i < len(text): + if text[i] == _ESCAPE_SENTINEL: + out.append("\\") + if i + 1 < len(text): + out.append(text[i + 1]) + i += 2 + else: + # Dangling sentinel (e.g. code span ending in `\`): the + # paired char became a delimiter; keep the backslash only. + i += 1 + continue + out.append(text[i]) + i += 1 + return "".join(out) + + +# Regex patterns for inline elements, ordered by priority. Each pattern +# captures a full inline construct. +# +# The patterns operate on text that has already been through +# `_protect_escapes`, so escaped delimiters appear as ``X`` +# (where SENTINEL = U+FDD0 -- the f-string ``﷐`` below). The +# ``(? ``\\``) -- are left +# as literal ``\`` and don't trigger the lookbehind, so the following +# delimiter is correctly recognised as a real opener (closing the +# doubled-backslash gap). +# +# Bracket content groups still need ``(?:[^\]﷐]|﷐.)+?`` so +# that escaped ``\]`` / ``\[`` inside link text don't prematurely +# terminate the match -- the alternation consumes a ``X`` +# pair as a single unit. +# +# Performance note: the alternation makes the link / image content +# match O(n^2) on adversarial inputs with many unclosed ``[`` (e.g. +# 5000 unclosed brackets -> ~1-2s parse). Typical chat content has a +# handful of brackets so this is sub-millisecond in practice. If a +# future surface ever feeds untrusted markdown with adversarial +# bracket counts, switch to a character-level walker for link/image +# content (the rest of `_parse_inline` is bounded by message size). _INLINE_PATTERNS = [ # Images: ![alt](url) or ![alt](url "title") - ("image", re.compile(r'!\[([^\]]*)\]\((\S+?)(?:\s+"([^"]*)")?\)')), + ("image", re.compile(r'(? str: + """Re-escape delimiter characters so a text leaf round-trips through + parse_markdown unchanged. + + Counterpart of :func:`_restore_escapes` -- a text node carrying the + value ``"*literal*"`` came from input ``\\*literal\\*``; stringify + must emit the backslashes or re-parse will form an emphasis node. + """ + if not any(c in _TEXT_DELIMITERS for c in value): + return value + out: list[str] = [] + for c in value: + if c in _TEXT_DELIMITERS: + out.append("\\") + out.append(c) + return "".join(out) + + +# Block-level constructs only fire at line start. When a paragraph's +# joined text begins with one of these patterns -- typically after +# `_restore_escapes` resolved a ``\#`` or ``\>`` to its literal char -- +# naive stringify would re-form the construct on the next parse. These +# regexes match the minimal opener pattern; `_escape_paragraph_block_markers` +# inserts the backslash at the right position. +_BLOCK_HEADING_RE = re.compile(r"^#{1,6}(?=\s|$)") +_BLOCK_BLOCKQUOTE_RE = re.compile(r"^>") +_BLOCK_UNORDERED_RE = re.compile(r"^[-+](?=\s)") +_BLOCK_ORDERED_RE = re.compile(r"^(\d+)([.)])(?=\s)") +_BLOCK_THEMATIC_DASH_RE = re.compile(r"^(-\s*){3,}\s*$") + + +def _escape_block_marker_line(line: str) -> str: + """Re-escape a block-level marker at the start of a single line.""" + if not line: + return line + if _BLOCK_HEADING_RE.match(line): + return "\\" + line + if _BLOCK_BLOCKQUOTE_RE.match(line): + return "\\" + line + if _BLOCK_UNORDERED_RE.match(line): + return "\\" + line + m = _BLOCK_ORDERED_RE.match(line) + if m: + # Escape the . or ) marker, not the digits -- `\1` isn't a + # CommonMark escape (digits aren't in escapable punct), so a + # leading-backslash placement wouldn't survive `_restore_escapes`. + return f"{m.group(1)}\\{m.group(2)}{line[m.end() :]}" + if _BLOCK_THEMATIC_DASH_RE.match(line): + return "\\" + line + return line + + +def _escape_paragraph_block_markers(text: str) -> str: + """Re-escape block-level markers at the start of every line in a + paragraph's joined output. + + Applied to the joined output of a `paragraph` node. Block-level + constructs (heading, blockquote, list, thematic break) only fire at + line start, so mid-line occurrences of ``#`` / ``>`` / ``-`` / ``+`` + are safe. But a paragraph text leaf may contain ``\\n`` (from + `_restore_escapes` resolving an escaped marker after a soft line + break, or from joining text leaves around a hard break); each line + of the joined output needs the same escape treatment as line 1, or + re-parse will split the paragraph at the embedded marker. + + ``*``-based and ``_``-based thematic breaks are already handled by + :func:`_escape_text_leaf` since both characters are in + `_TEXT_DELIMITERS`. + """ + if not text: + return text + if "\n" not in text: + return _escape_block_marker_line(text) + return "\n".join(_escape_block_marker_line(line) for line in text.split("\n")) + + def _parse_inline_plain(text: str) -> list[Content]: """Parse plain text that contains no inline formatting. - Handles hard line breaks (two trailing spaces + newline). + Handles hard line breaks (two trailing spaces + newline) and + restores escape-sentinel pairs to their literal character. The + caller is expected to have run ``_protect_escapes`` on the input. """ if " \n" in text: parts: list[Content] = [] segments = text.split(" \n") for i, seg in enumerate(segments): if seg: - parts.append(make_text(seg)) + parts.append(make_text(_restore_escapes(seg))) if i < len(segments) - 1: parts.append(make_break()) - return parts if parts else [make_text(text)] - return [make_text(text)] + return parts if parts else [make_text(_restore_escapes(text))] + return [make_text(_restore_escapes(text))] -def _parse_inline(text: str) -> list[Content]: +def _parse_inline(text: str, *, _already_protected: bool = False) -> list[Content]: """Parse inline markdown elements into AST nodes. Handles: strong, emphasis, delete, inlineCode, link, image. Returns a list of inline Content nodes. + Runs ``_protect_escapes`` on entry so the inline regex patterns can + use sentinel-based lookbehinds. Recursive calls (e.g. into link + text or emphasis content) pass ``_already_protected=True`` because + the captured group is already in the protected form. + The suffix (text after a match) is processed iteratively to avoid unbounded recursion on long strings. Content *inside* a match (e.g. bold text, link text) still recurses, but that depth is @@ -214,6 +457,8 @@ def _parse_inline(text: str) -> list[Content]: """ if not text: return [] + if not _already_protected: + text = _protect_escapes(text) nodes: list[Content] = [] remaining = text @@ -242,23 +487,30 @@ def _parse_inline(text: str) -> list[Content]: # The matched construct (content recursion is bounded by match length) if best_kind == "image": - alt = best_match.group(1) - url = best_match.group(2) + alt = _restore_escapes(best_match.group(1)) + url = _restore_escapes(best_match.group(2)) title = best_match.group(3) + if title is not None: + title = _restore_escapes(title) nodes.append(make_image(url, alt, title)) elif best_kind == "link": link_text = best_match.group(1) - url = best_match.group(2) + url = _restore_escapes(best_match.group(2)) title = best_match.group(3) - nodes.append(make_link(url, _parse_inline(link_text), title)) + if title is not None: + title = _restore_escapes(title) + nodes.append(make_link(url, _parse_inline(link_text, _already_protected=True), title)) elif best_kind == "inlineCode": - nodes.append(make_inline_code(best_match.group(1))) + # CommonMark: backslashes inside inline code are literal, + # so restore sentinel pairs as `\X` rather than dropping the + # backslash. + nodes.append(make_inline_code(_restore_escapes_as_literal_pair(best_match.group(1)))) elif best_kind in ("strong_star", "strong_under"): - nodes.append(make_strong(_parse_inline(best_match.group(1)))) + nodes.append(make_strong(_parse_inline(best_match.group(1), _already_protected=True))) elif best_kind == "delete": - nodes.append(make_delete(_parse_inline(best_match.group(1)))) + nodes.append(make_delete(_parse_inline(best_match.group(1), _already_protected=True))) elif best_kind in ("emphasis_star", "emphasis_under"): - nodes.append(make_emphasis(_parse_inline(best_match.group(1)))) + nodes.append(make_emphasis(_parse_inline(best_match.group(1), _already_protected=True))) # Advance past the match (iterative, not recursive) remaining = remaining[best_match.end() :] @@ -319,6 +571,11 @@ def _collect_list_items(lines: list[str], start: int, ordered: bool) -> tuple[li items: list[Content] = [] i = start item_re = re.compile(r"^(\d+)[.)]\s+(.*)") if ordered else re.compile(r"^[-*+]\s+(.*)") + # GFM task list extension: `- [ ]` or `- [x]` at the start of an + # unordered item maps to `listItem.checked = False` / `True`. The + # trailing whitespace + content group is optional so an empty task + # item (``- [ ]`` with nothing after) still produces ``checked``. + task_re = re.compile(r"^\[([ xX])\](?:\s+(.*))?$") while i < len(lines): line = lines[i] @@ -326,6 +583,15 @@ def _collect_list_items(lines: list[str], start: int, ordered: bool) -> tuple[li if m: item_text = m.group(2) if ordered else m.group(1) + checked: bool | None = None + if not ordered: + task_m = task_re.match(item_text) + if task_m: + checked = task_m.group(1) in ("x", "X") + # group(2) is None for an empty task item (`- [ ]`) + # with no trailing whitespace/content. + item_text = task_m.group(2) or "" + item_children_lines = [item_text] i += 1 @@ -350,7 +616,7 @@ def _collect_list_items(lines: list[str], start: int, ordered: bool) -> tuple[li # Parse nested content in the item nested_children = _parse_list_item_content(item_children_lines) - items.append(make_list_item(nested_children)) + items.append(make_list_item(nested_children, checked=checked)) else: break @@ -579,14 +845,17 @@ def _stringify_node(node: Content, *, emphasis: str = "*", bullet: str = "*") -> node_type = node.get("type") if node_type == "text": - return node.get("value", "") + return _escape_text_leaf(node.get("value", "")) if node_type == "break": - return "\n" + # Hard line break in CommonMark is two trailing spaces + newline. + # Emitting just `\n` would degrade to a soft break on re-parse. + return " \n" if node_type == "paragraph": children = node.get("children", []) - return "".join(_stringify_node(c, emphasis=emphasis, bullet=bullet) or "" for c in children) + joined = "".join(_stringify_node(c, emphasis=emphasis, bullet=bullet) or "" for c in children) + return _escape_paragraph_block_markers(joined) if node_type == "heading": depth = node.get("depth", 1) @@ -627,7 +896,7 @@ def _stringify_node(node: Content, *, emphasis: str = "*", bullet: str = "*") -> return f"[{text}]({url})" if node_type == "image": - alt = node.get("alt", "") + alt = _escape_text_leaf(node.get("alt", "")) url = node.get("url", "") title = node.get("title") if title: @@ -641,21 +910,52 @@ def _stringify_node(node: Content, *, emphasis: str = "*", bullet: str = "*") -> lines: list[str] = [] for idx, item in enumerate(items): prefix = f"{start + idx}." if ordered else bullet + # GFM task-list marker, only on unordered items with `checked` set. + checked = item.get("checked") if not ordered else None + task_marker = "" + if checked is True: + task_marker = "[x] " + elif checked is False: + task_marker = "[ ] " item_children = item.get("children", []) - for ci, child in enumerate(item_children): + if not item_children: + # Empty item -- still emit the prefix (and any task marker) + # so the round-trip preserves an empty `- [ ]` task. + lines.append(f"{prefix} {task_marker}".rstrip()) + continue + # Track whether the parent prefix has been written so a + # listItem whose only children are nested lists still gets + # its own line (PR #101 review finding). + prefix_emitted = False + for child in item_children: child_type = child.get("type") if child_type == "list": + if not prefix_emitted: + # No text child before this nested list -- emit + # the parent prefix on its own line so the + # listItem and any checkbox state survive. + lines.append(f"{prefix} {task_marker}".rstrip()) + prefix_emitted = True nested = _stringify_node(child, emphasis=emphasis, bullet=bullet) if nested: - # Indent nested list for nl in nested.split("\n"): lines.append(f" {nl}") else: text = _stringify_node(child, emphasis=emphasis, bullet=bullet) or "" - if ci == 0: - lines.append(f"{prefix} {text}") + # A paragraph's stringified text may contain `\n` from + # soft breaks or hard breaks. Continuation lines must + # be indented by 2 spaces so the round-trip stays + # inside the list item rather than splitting back out + # to a sibling paragraph. + text_lines = text.split("\n") + if not prefix_emitted: + lines.append(f"{prefix} {task_marker}{text_lines[0]}") + for cont in text_lines[1:]: + lines.append(f" {cont}" if cont else "") + prefix_emitted = True else: - lines.append(f" {text}") + for cont in text_lines: + lines.append(f" {cont}" if cont else "") return "\n".join(lines) if node_type == "listItem": diff --git a/src/chat_sdk/shared/streaming_markdown.py b/src/chat_sdk/shared/streaming_markdown.py index 6a4f880..8b4ae61 100644 --- a/src/chat_sdk/shared/streaming_markdown.py +++ b/src/chat_sdk/shared/streaming_markdown.py @@ -54,6 +54,60 @@ def _strip_fenced_code(text: str) -> str: return "\n".join(result_lines) +# ASCII-punctuation characters that can be backslash-escaped per CommonMark. +# When `\X` precedes one of these, the resulting char is literal, not a +# delimiter -- so the inline-marker counters in `_remend` should ignore it. +_ESCAPABLE_PUNCT = frozenset("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") + + +def _strip_escape_sequences(text: str) -> str: + """Remove ``\\X`` pairs from *text* so escape-aware counters work. + + For each ``\\X`` where X is an ASCII punctuation char, drop both + characters from the result. The cleaned text is only used for + *counting* delimiters (``*``, ``~~``, `` ` ``, ``[``, ``]``); + positions in the cleaned text don't map back to the original, so + truncating is safe. + """ + if "\\" not in text: + return text + out: list[str] = [] + i = 0 + while i < len(text): + if text[i] == "\\" and i + 1 < len(text) and text[i + 1] in _ESCAPABLE_PUNCT: + i += 2 + continue + out.append(text[i]) + i += 1 + return "".join(out) + + +def _strip_math_regions(text: str) -> str: + """Replace ``$$...$$`` and ``$...$`` math spans with empty strings. + + LLMs frequently emit TeX-style math in technical chat content. Inline + markers inside math (e.g. ``$a^* + b^*$``) are literal, not delimiters, + so they shouldn't count toward the unclosed-emphasis tally. Math is + line-scoped here -- a ``$`` that doesn't close on the same line is + left alone so it doesn't gobble unrelated prose. The double-dollar + pass runs first so ``$$`` blocks don't get mis-paired by the single + pass. + + Inline math requires a non-whitespace char immediately after the + opening ``$`` and immediately before the closing ``$`` (standard + pandoc/markdown-it heuristic). This prevents false positives on + currency text like ``"prices are $5 and $10"`` which would otherwise + pair up the two dollar signs as math (PR #101 review finding). + """ + if "$" not in text: + return text + # Display math: $$...$$ may span multiple lines. + text = re.sub(r"\$\$.+?\$\$", "", text, flags=re.DOTALL) + # Inline math: $X...Y$ on a single line where X and Y are non-whitespace. + text = re.sub(r"\$(?!\s)[^\$\n]+?(? bool: """Whether a single-``*`` run at *run_start* should be excluded from the emphasis tally per CommonMark's flanking rules. @@ -63,6 +117,12 @@ def _is_excluded_asterisk(stripped: str, run_start: int, run_len: int, ch: str) covers line-leading bullet list markers (``* item``). Mirrors ``shouldSkipAsterisk`` in upstream remend's ``emphasis-handlers.ts``. + Note: a simpler word-internal exclusion (e.g. ``5*3=15``) was + considered but breaks paired emphasis like ``text*foo*`` whose + asterisks are also word-flanked. Distinguishing the two needs a + proper CommonMark delimiter-stack algorithm, tracked under issue #69 + Option A as the next chat-completeness item. + Only single-``*`` runs are subject to this check -- ``**`` / ``***`` runs are evaluated by the bold pairing logic and have their own word-boundary semantics. ``_``-emphasis isn't ambiguous with list @@ -180,6 +240,14 @@ def _remend(text: str) -> str: # Strip fenced code blocks so their contents don't affect inline counts. outside_fences = _strip_fenced_code(result) + # Drop ASCII-punct escape sequences BEFORE math-region stripping -- + # otherwise an escaped ``\$`` could pair with a later unescaped ``$`` + # and the inline-math regex would eat content between them, including + # any emphasis openers (PR #101 review finding). + outside_fences = _strip_escape_sequences(outside_fences) + # Drop math regions ($...$ and $$...$$) -- markers inside math are + # literal, not delimiters. + outside_fences = _strip_math_regions(outside_fences) # --- inline code backticks --- # Count total backtick characters outside code fences. If odd, one code diff --git a/tests/test_markdown_faithful.py b/tests/test_markdown_faithful.py index f5ad5c9..c1b326c 100644 --- a/tests/test_markdown_faithful.py +++ b/tests/test_markdown_faithful.py @@ -189,6 +189,220 @@ def test_handles_multiple_paragraphs(self): assert ast["children"][1]["type"] == "paragraph" +# ============================================================================ +# Chat / AI agent completeness (issue #69): escaped chars + task lists +# ============================================================================ + + +class TestEscapedCharacters: + """Backslash-escape support for chat content. + + LLMs routinely emit ``\\*`` / ``\\[`` / ``\\\\`` when explaining + syntax, displaying shell commands, or showing literal markdown. + Previously these slipped past the inline regexes and the visible + output dropped the backslashes but mis-parsed the surrounding + delimiters as emphasis / links. + """ + + def _para_children(self, md): + ast = parse_markdown(md) + assert ast["children"][0]["type"] == "paragraph" + return ast["children"][0]["children"] + + def test_escaped_asterisk_is_literal_not_italic(self): + children = self._para_children(r"\*not italic\*") + assert len(children) == 1 + assert children[0] == {"type": "text", "value": "*not italic*"} + + def test_escaped_underscore_is_literal_not_italic(self): + children = self._para_children(r"\_not italic\_") + assert len(children) == 1 + assert children[0] == {"type": "text", "value": "_not italic_"} + + def test_escaped_brackets_do_not_form_a_link(self): + children = self._para_children(r"\[not a link\](https://x.com)") + assert all(c.get("type") != "link" for c in children) + assert any("[not a link]" in c.get("value", "") for c in children if c.get("type") == "text") + + def test_escaped_tilde_does_not_form_strikethrough(self): + children = self._para_children(r"\~~not strike\~~") + assert all(c.get("type") != "delete" for c in children) + # Also pin the actual text content -- a buggy impl that dropped + # the tildes entirely would pass the absence-of-`delete` check. + text_concat = "".join(c.get("value", "") for c in children if c.get("type") == "text") + assert "~~not strike~~" in text_concat + + def test_escaped_backslash_yields_single_backslash(self): + children = self._para_children(r"path with \\backslash") + assert children == [{"type": "text", "value": "path with \\backslash"}] + + def test_mixed_escaped_and_real_emphasis(self): + children = self._para_children(r"\*literal\* and *real italic*") + types = [c.get("type") for c in children] + assert "emphasis" in types + text_values = [c.get("value", "") for c in children if c.get("type") == "text"] + assert any("*literal*" in v for v in text_values) + + def test_escaped_alt_in_image(self): + children = self._para_children(r"![my \[image\]](pic.png)") + assert children[0]["type"] == "image" + assert children[0]["alt"] == "my [image]" + + def test_link_url_resolves_backslash_escapes(self): + # PR #101 review #3 (downstream-impact reviewer): the inline + # parser now resolves backslash escapes in link URLs per + # CommonMark spec. This is a behavior change from the previous + # raw pass-through; pin it so the contract is explicit. + children = self._para_children(r"[t](http://example.com/path\)more)") + link_nodes = [c for c in children if c.get("type") == "link"] + # The `\)` inside the URL is now resolved to literal `)`. + # Previously this URL would have been truncated at the first `)`. + assert len(link_nodes) == 1 + # Escaped `*` in URL becomes literal `*` (rare but spec-correct). + children2 = self._para_children(r"[t](u\*r\*l)") + link_nodes2 = [c for c in children2 if c.get("type") == "link"] + assert len(link_nodes2) == 1 + assert link_nodes2[0]["url"] == "u*r*l" + + def test_inline_code_contents_are_not_unescaped(self): + # Per CommonMark, backslash inside `code` is literal. + children = self._para_children(r"a `\*literal\*` b") + code_nodes = [c for c in children if c.get("type") == "inlineCode"] + assert len(code_nodes) == 1 + assert code_nodes[0]["value"] == r"\*literal\*" + + def test_code_span_ending_in_backslash_does_not_leak_sentinel(self): + # PR #101 re-review (holistic): a code span ending in `\` (e.g. + # `` `C:\` ``) used to leave the escape sentinel (U+FDD0) + # dangling in the captured content, leaking the noncharacter + # into output. The dangling sentinel now resolves to a literal + # backslash (CommonMark: backslash is literal inside code spans). + for src, expected_code in [ + ("Use `C:\\` as root", "C:\\"), + ("regex `\\d+\\` here", "\\d+\\"), + ("a `b\\` c", "b\\"), + ]: + children = self._para_children(src) + code_nodes = [c for c in children if c.get("type") == "inlineCode"] + assert len(code_nodes) == 1, f"expected 1 code span in {src!r}" + assert code_nodes[0]["value"] == expected_code + # The sentinel codepoint must never reach output. + assert "﷐" not in code_nodes[0]["value"] + + def test_escaped_backslash_followed_by_real_emphasis(self): + # PR #101 review #4 (chatgpt-codex P2): ``\\*foo*`` is literal + # backslash + italic. The fixed-width ``(?`, `\-`, etc. at paragraph start, stringify + # must re-emit the backslash or re-parse promotes the paragraph + # into the corresponding block construct. + for src in [ + r"\# not heading", + r"\> not quote", + r"\- not list", + r"\+ also not list", + r"1\. not ordered", + r"2\) other ordered", + r"\* not bullet", + r"\---", + ]: + ast = parse_markdown(src) + ast2 = parse_markdown(stringify_markdown(ast)) + assert ast == ast2, f"paragraph-start drift on {src!r}" + + def test_stringify_does_not_over_escape_mid_text_block_markers(self): + # `#` / `>` / `-` / digits-with-dot are only block markers at + # line start. Mid-text they're literal -- shouldn't be escaped. + for src in [ + "see > arrow operator", + "Step 1. Then 2. Then 3.", + "use - dash here", + "ordinary # hash inline", + ]: + out = stringify_markdown(parse_markdown(src)) + assert "\\" not in out, f"over-escape on {src!r}: {out!r}" + + def test_roundtrip_preserves_escaped_block_markers_after_soft_break(self): + # A paragraph text leaf can contain `\n` (from a soft break + a + # following line that doesn't qualify as a block construct + # because the marker was originally escaped). The line-2 marker + # also needs re-escaping or stringify drops it. + for src in [ + "Here is a paragraph\n\\# 1 is not a heading", + "first line\n\\> not quote", + "line one\n\\- not list", + "alpha\n1\\. not ordered", + "alpha\n\\---", + ]: + ast = parse_markdown(src) + ast2 = parse_markdown(stringify_markdown(ast)) + assert ast == ast2, f"soft-break drift on {src!r}" + + def test_roundtrip_preserves_list_item_continuation_indent(self): + # PR #101 adversarial review: a multi-line list item like + # `- item1\n para2` (LLMs frequently emit wrapped bullets) + # parses to one list with one item containing one paragraph + # `item1\npara2`. The stringifier must indent the continuation + # line by 2 spaces or re-parse drops out of the list and + # produces `[list, paragraph]`. + for src in [ + "- item1\n para2", + "- a\n b\n c", + "- first wrapped\n second line\n- second item", + "1. ordered\n continuation", + ]: + ast = parse_markdown(src) + ast2 = parse_markdown(stringify_markdown(ast)) + assert ast == ast2, f"list-item continuation drift on {src!r}" + + def test_roundtrip_preserves_hard_break_in_paragraph(self): + # PR #101 adversarial review: a `break` node (hard line break) + # previously stringified to a bare `\n`, which re-parses as a + # soft break -- collapsing `[text, break, text]` into a single + # text leaf with embedded `\n`. Must emit `" \n"` per CommonMark. + ast = parse_markdown("Step \nline2") + ast2 = parse_markdown(stringify_markdown(ast)) + assert ast == ast2 + # And the children should still include a `break` node, not + # just a single merged text leaf. + para = ast2["children"][0] + types = [c.get("type") for c in para["children"]] + assert "break" in types + # ============================================================================ # toPlainText Tests diff --git a/tests/test_streaming_markdown.py b/tests/test_streaming_markdown.py index 392a55a..38afddb 100644 --- a/tests/test_streaming_markdown.py +++ b/tests/test_streaming_markdown.py @@ -1089,3 +1089,83 @@ def test_second_table_after_blank_line_still_holds_header(self): committable = r.get_committable_text() assert "| X | Y |" not in committable assert "| 1 | 2 |" in committable + + +# ============================================================================ +# Chat / AI agent completeness (issue #69 follow-up) +# ============================================================================ + + +class TestRemendChatCompleteness: + """``_remend`` parity with upstream remend on chat-content patterns. + + Each gap below comes from the [issue #69 follow-up + catalog](https://github.com/Chinchill-AI/chat-sdk-python/issues/69#issuecomment-4514898801). + Word-internal asterisks (``5*3=15``) are tracked separately under + Option A -- they need a proper CommonMark delimiter-stack algorithm + to avoid breaking paired-emphasis cases like ``text*foo*``. + """ + + # --- Math regions ($...$, $$...$$) ----------------------------------- + + def test_remend_skips_markers_inside_inline_math(self): + assert _remend("$a^* + b^*$") == "$a^* + b^*$" + + def test_remend_skips_markers_inside_display_math(self): + text = "$$\\int_0^* e^{-x} dx$$" + assert _remend(text) == text + + def test_remend_still_closes_emphasis_outside_math(self): + # Math region is stripped for counting; the trailing italic still + # gets closed normally. + assert _remend("$a^2$ and *italic") == "$a^2$ and *italic*" + + def test_remend_skips_strike_marker_inside_math(self): + assert _remend("$a ~~ b$") == "$a ~~ b$" + + def test_remend_does_not_open_bracket_inside_math(self): + # Math regions are dropped from the bracket walk too -- a literal + # `[` inside math shouldn't add a phantom `]` closer. + assert _remend("note $f[x]$ here") == "note $f[x]$ here" + + # --- Escape-aware tilde / backtick / bracket counters ---------------- + + def test_remend_does_not_add_strike_for_escaped_tilde_pair(self): + assert _remend(r"foo \~~bar") == r"foo \~~bar" + + def test_remend_does_not_add_bracket_for_escaped_open(self): + assert _remend(r"see \[item") == r"see \[item" + + def test_remend_does_not_add_backtick_for_escaped(self): + assert _remend(r"foo \` bar") == r"foo \` bar" + + def test_remend_handles_mixed_escaped_and_real_unclosed(self): + # `\~~` is literal; the trailing real `~~` is unclosed -> close it. + assert _remend(r"a \~~b and ~~real") == r"a \~~b and ~~real~~" + + def test_remend_does_not_affect_escape_outside_relevant_counters(self): + # An escaped delimiter that wasn't going to imbalance anything + # is still left untouched. + assert _remend(r"foo \* bar") == r"foo \* bar" + + # --- Escape-before-math ordering (PR #101 review #1) --------------------- + + def test_remend_escaped_dollar_does_not_pair_with_unescaped_dollar(self): + # Without the escape-strip-before-math-strip ordering, the math + # regex would pair these two `$`s and eat the `*` opener inside, + # leaving italic unclosed. + text = r"\$opener *unclosed text closer\$" + assert _remend(text) == text + "*" + + def test_remend_escaped_dollar_does_not_create_phantom_math_region(self): + # `\$5` is a literal dollar amount; the `$10` later is not part of + # any math region (one un-escaped `$` doesn't form `$...$`). + # The italic at the end still gets closed normally. + assert _remend(r"\$5 and $10 *italic") == r"\$5 and $10 *italic*" + + def test_remend_unescaped_currency_does_not_pair_as_math(self): + # PR #101 review #1: text like `prices are $5 and $10` would + # previously match `$5 and $10` as inline math (because the + # regex didn't require non-whitespace around the delimiters). + # The italic at end must still get closed normally. + assert _remend("prices are $5 and $10 *italic") == "prices are $5 and $10 *italic*"