From 18e4ceb490662073bdeb611f2949c0bcb7960a86 Mon Sep 17 00:00:00 2001
From: Martin Vogel <martin.vogel.tech@gmail.com>
Date: Thu, 16 Jul 2026 00:53:22 +0200
Subject: [PATCH] fix(extract): truncate comments/docstrings at UTF-8 character
 boundaries
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

extract_comment_text capped over-long comments with a raw byte cut
(text[MAX_COMMENT_LEN] = 0), which splits a multi-byte UTF-8 character
straddling byte 500 and leaves an incomplete sequence. That corrupted
value lands verbatim in nodes.properties — syntactically valid JSON but
invalid UTF-8 — so any consumer decoding the SQLite TEXT column strictly
(Python's sqlite3, the default for tooling on the exported DB) throws on
read. It never surfaced on English-only code because a single-byte-per-
char comment can be cut anywhere safely; it reproduces on Japanese/
Chinese/etc. comments (reporter: EC-CUBE).

Snap the cut back over UTF-8 continuation bytes (10xxxxxx) to the lead
byte of the straddling character and terminate there, dropping the
partial character. Result is always valid UTF-8.

Reproduce-first: python_docstring_utf8_boundary_truncation_issue1017
builds a docstring with 3-byte CJK chars (E3 81 82) straddling the cap
and asserts the captured docstring decodes as valid UTF-8 (via an
inline validator). RED before (mid-char split), GREEN after.

Closes #1017

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
---
 internal/cbm/extract_defs.c | 13 +++++++-
 tests/test_extraction.c     | 66 +++++++++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+), 1 deletion(-)

diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c
index deee8a8e7..03637d709 100644
--- a/internal/cbm/extract_defs.c
+++ b/internal/cbm/extract_defs.c
@@ -1192,7 +1192,18 @@ static bool is_comment_node(const char *kind) {
 static char *extract_comment_text(CBMArena *a, TSNode node, const char *source) {
     char *text = cbm_node_text(a, node, source);
     if (text && strlen(text) > MAX_COMMENT_LEN) {
-        text[MAX_COMMENT_LEN] = '\0';
+        /* #1017: snap the cut back to a UTF-8 character boundary so a
+         * multi-byte character straddling MAX_COMMENT_LEN (e.g. a 3-byte
+         * CJK char in a Japanese docstring) is never split — a byte-offset
+         * cut leaves an incomplete sequence that lands as invalid UTF-8 in
+         * nodes.properties, and strict decoders (Python's sqlite3) throw on
+         * read. Continuation bytes are 10xxxxxx; back up over them to the
+         * lead byte, dropping the partial character. */
+        size_t cut = MAX_COMMENT_LEN;
+        while (cut > 0 && ((unsigned char)text[cut] & 0xC0) == 0x80) {
+            cut--;
+        }
+        text[cut] = '\0';
     }
     return text;
 }
diff --git a/tests/test_extraction.c b/tests/test_extraction.c
index 0598cadb7..1c6bf02bd 100644
--- a/tests/test_extraction.c
+++ b/tests/test_extraction.c
@@ -1917,6 +1917,71 @@ TEST(wolfram_nested_def) {
  * Group I: cbm_test.go ports
  * ═══════════════════════════════════════════════════════════════════ */
 
+/* #1017: a docstring longer than MAX_COMMENT_LEN (500) that has a multi-byte
+ * UTF-8 character straddling the cap must be truncated at a CHARACTER
+ * boundary, never mid-sequence — a byte-offset cut leaves an incomplete
+ * UTF-8 sequence that lands as invalid UTF-8 in nodes.properties and makes
+ * strict decoders (Python sqlite3) throw. Verified by decoding the captured
+ * docstring as UTF-8. */
+static int is_valid_utf8(const char *s) {
+    const unsigned char *p = (const unsigned char *)s;
+    while (*p) {
+        int n;
+        if (*p < 0x80) {
+            n = 0;
+        } else if ((*p & 0xE0) == 0xC0) {
+            n = 1;
+        } else if ((*p & 0xF0) == 0xE0) {
+            n = 2;
+        } else if ((*p & 0xF8) == 0xF0) {
+            n = 3;
+        } else {
+            return 0; /* invalid lead byte (or stray continuation) */
+        }
+        p++;
+        for (int i = 0; i < n; i++) {
+            if ((*p & 0xC0) != 0x80) {
+                return 0; /* missing/short continuation → truncated mid-char */
+            }
+            p++;
+        }
+    }
+    return 1;
+}
+
+TEST(python_docstring_utf8_boundary_truncation_issue1017) {
+    /* 495 ASCII bytes then 3-byte CJK chars (E3 81 82 = U+3042 あ): the cap at
+     * byte 500 lands in the middle of the second あ. */
+    char src[900];
+    int off = snprintf(src, sizeof(src), "def f():\n    \"\"\"");
+    for (int i = 0; i < 495; i++) {
+        src[off++] = 'x';
+    }
+    for (int i = 0; i < 5; i++) {
+        src[off++] = (char)0xE3;
+        src[off++] = (char)0x81;
+        src[off++] = (char)0x82;
+    }
+    off += snprintf(src + off, sizeof(src) - off, "\"\"\"\n    return 1\n");
+    (void)off;
+    CBMFileResult *r = extract(src, CBM_LANG_PYTHON, "test", "test.py");
+    ASSERT_NOT_NULL(r);
+    const char *doc = NULL;
+    for (int i = 0; i < r->defs.count; i++) {
+        if (r->defs.items[i].name && strcmp(r->defs.items[i].name, "f") == 0) {
+            doc = r->defs.items[i].docstring;
+        }
+    }
+    ASSERT_NOT_NULL(doc);
+    if (!is_valid_utf8(doc)) {
+        fprintf(stderr, "  [1017] FAIL docstring truncated mid-UTF-8-char (len=%zu)\n",
+                strlen(doc));
+    }
+    ASSERT_TRUE(is_valid_utf8(doc));
+    cbm_free_result(r);
+    PASS();
+}
+
 TEST(python_docstring) {
     CBMFileResult *r = extract(
         "def compute(x, y):\n    \"\"\"Compute the sum of x and y.\"\"\"\n    return x + y\n",
@@ -4723,6 +4788,7 @@ SUITE(extraction) {
     RUN_TEST(wolfram_nested_def);
 
     /* cbm_test.go ports */
+    RUN_TEST(python_docstring_utf8_boundary_truncation_issue1017);
     RUN_TEST(python_docstring);
     RUN_TEST(go_function_extraction);
     RUN_TEST(js_arrow_function);