Handle escape sequences in backtick identifiers and sanitize invalid …

…UTF-8

1. Lexer: Process escape sequences (\xFF, \0, etc.) in backtick-quoted
   identifiers, matching the behavior of string literals.

2. Explain output:
   - Replace invalid UTF-8 bytes with replacement character (U+FFFD)
   - Display null bytes as escape sequence \0
   - Escape backslashes and single quotes in identifier/alias output

3. Apply sanitization to:
   - Column declarations
   - Identifier names
   - Function aliases
   - Storage definition ORDER BY identifiers

This matches ClickHouse's EXPLAIN AST behavior for handling special
characters in identifiers.

Fixes test: 03356_tables_with_binary_identifiers_invalid_utf8/stmt2

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

internal/explain/explain.go

@@ -350,9 +350,9 @@ func Column(sb *strings.Builder, col *ast.ColumnDeclaration, depth int) {
 		children++
 	}
 	if children > 0 {
-		fmt.Fprintf(sb, "%sColumnDeclaration %s (children %d)\n", indent, col.Name, children)
+		fmt.Fprintf(sb, "%sColumnDeclaration %s (children %d)\n", indent, sanitizeUTF8(col.Name), children)
 	} else {
-		fmt.Fprintf(sb, "%sColumnDeclaration %s\n", indent, col.Name)
+		fmt.Fprintf(sb, "%sColumnDeclaration %s\n", indent, sanitizeUTF8(col.Name))
 	}
 	if col.Type != nil {
 		Node(sb, col.Type, depth+1)

internal/explain/expressions.go

@@ -4,10 +4,48 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/sqlc-dev/doubleclick/ast"
 )
 
+// sanitizeUTF8 replaces invalid UTF-8 bytes with the Unicode replacement character (U+FFFD)
+// and null bytes with the escape sequence \0.
+// This matches ClickHouse's behavior of displaying special bytes in EXPLAIN AST output.
+func sanitizeUTF8(s string) string {
+	// Check if we need to process at all
+	needsProcessing := !utf8.ValidString(s)
+	if !needsProcessing {
+		for i := 0; i < len(s); i++ {
+			if s[i] == 0 {
+				needsProcessing = true
+				break
+			}
+		}
+	}
+	if !needsProcessing {
+		return s
+	}
+
+	var result strings.Builder
+	for i := 0; i < len(s); {
+		r, size := utf8.DecodeRuneInString(s[i:])
+		if r == utf8.RuneError && size == 1 {
+			// Invalid byte - write replacement character
+			result.WriteRune('\uFFFD')
+			i++
+		} else if r == 0 {
+			// Null byte - write as escape sequence \0
+			result.WriteString("\\0")
+			i += size
+		} else {
+			result.WriteRune(r)
+			i += size
+		}
+	}
+	return result.String()
+}
+
 // escapeAlias escapes backslashes and single quotes in alias names for EXPLAIN output
 func escapeAlias(alias string) string {
 	// Escape backslashes first, then single quotes
@@ -25,21 +63,31 @@ func explainIdentifier(sb *strings.Builder, n *ast.Identifier, indent string) {
 	}
 }
 
-// formatIdentifierName formats an identifier name, handling JSON path notation
+// escapeIdentifierPart escapes backslashes and single quotes in an identifier part
+// and sanitizes invalid UTF-8 bytes
+func escapeIdentifierPart(s string) string {
+	s = sanitizeUTF8(s)
+	s = strings.ReplaceAll(s, "\\", "\\\\")
+	s = strings.ReplaceAll(s, "'", "\\'")
+	return s
+}
+
+// formatIdentifierName formats an identifier name, handling JSON path notation,
+// sanitizing invalid UTF-8 bytes, and escaping special characters
 func formatIdentifierName(n *ast.Identifier) string {
 	if len(n.Parts) == 0 {
 		return ""
 	}
 	if len(n.Parts) == 1 {
-		return n.Parts[0]
+		return escapeIdentifierPart(n.Parts[0])
 	}
-	result := n.Parts[0]
+	result := escapeIdentifierPart(n.Parts[0])
 	for _, p := range n.Parts[1:] {
 		// JSON path notation: ^fieldname should be formatted as ^`fieldname`
 		if strings.HasPrefix(p, "^") {
-			result += ".^`" + p[1:] + "`"
+			result += ".^`" + escapeIdentifierPart(p[1:]) + "`"
 		} else {
-			result += "." + p
+			result += "." + escapeIdentifierPart(p)
 		}
 	}
 	return result

internal/explain/functions.go

@@ -7,11 +7,11 @@ import (
 	"github.com/sqlc-dev/doubleclick/ast"
 )
 
-// escapeFunctionAlias escapes single quotes in function alias names.
-// Unlike escapeAlias (for column aliases), this does NOT escape backslashes
-// since ClickHouse EXPLAIN AST preserves backslashes in function aliases.
+// escapeFunctionAlias escapes backslashes and single quotes in function alias names.
+// This is needed because the lexer processes escape sequences in backtick identifiers.
 func escapeFunctionAlias(alias string) string {
-	return strings.ReplaceAll(alias, "'", "\\'")
+	result := strings.ReplaceAll(alias, "\\", "\\\\")
+	return strings.ReplaceAll(result, "'", "\\'")
 }
 
 // normalizeIntervalUnit converts interval units to title-cased singular form

internal/explain/statements.go

@@ -460,9 +460,9 @@ func explainCreateQuery(sb *strings.Builder, n *ast.CreateQuery, indent string,
 					// When ORDER BY has modifiers (ASC/DESC), wrap in StorageOrderByElement
 					if n.OrderByHasModifiers {
 						fmt.Fprintf(sb, "%s StorageOrderByElement (children %d)\n", storageIndent, 1)
-						fmt.Fprintf(sb, "%s  Identifier %s\n", storageIndent, ident.Name())
+						fmt.Fprintf(sb, "%s  Identifier %s\n", storageIndent, sanitizeUTF8(ident.Name()))
 					} else {
-						fmt.Fprintf(sb, "%s Identifier %s\n", storageIndent, ident.Name())
+						fmt.Fprintf(sb, "%s Identifier %s\n", storageIndent, sanitizeUTF8(ident.Name()))
 					}
 				} else if lit, ok := n.OrderBy[0].(*ast.Literal); ok && lit.Type == ast.LiteralTuple {
 					// Handle tuple literal - for ORDER BY with modifiers (DESC/ASC),

lexer/lexer.go

@@ -698,6 +698,63 @@ func (l *Lexer) readBacktickIdentifier() Item {
 			l.readChar() // skip closing backtick
 			break
 		}
+		if l.ch == '\\' {
+			l.readChar() // consume backslash
+			if l.eof {
+				break
+			}
+			// Interpret escape sequence (same as readString)
+			switch l.ch {
+			case '\'':
+				sb.WriteRune('\'')
+			case '"':
+				sb.WriteRune('"')
+			case '\\':
+				sb.WriteRune('\\')
+			case '`':
+				sb.WriteRune('`')
+			case 'n':
+				sb.WriteRune('\n')
+			case 't':
+				sb.WriteRune('\t')
+			case 'r':
+				sb.WriteRune('\r')
+			case '0':
+				sb.WriteRune('\x00')
+			case 'a':
+				sb.WriteRune('\a')
+			case 'b':
+				sb.WriteRune('\b')
+			case 'f':
+				sb.WriteRune('\f')
+			case 'v':
+				sb.WriteRune('\v')
+			case 'e':
+				sb.WriteRune('\x1b') // escape character (ASCII 27)
+			case 'x':
+				// Hex escape: \xNN
+				l.readChar()
+				if l.eof {
+					break
+				}
+				hex1 := l.ch
+				l.readChar()
+				if l.eof {
+					sb.WriteRune(rune(hexValue(hex1)))
+					continue
+				}
+				hex2 := l.ch
+				// Convert hex digits to byte
+				val := hexValue(hex1)*16 + hexValue(hex2)
+				sb.WriteByte(byte(val))
+			default:
+				// Unknown escape, preserve both the backslash and the character
+				sb.WriteRune('\\')
+				sb.WriteRune(l.ch)
+			}
+			l.readChar()
+			continue
+		}
 		sb.WriteRune(l.ch)
 		l.readChar()
 	}

parser/testdata/03356_tables_with_binary_identifiers_invalid_utf8/metadata.json

@@ -1,5 +1 @@
-{
-  "explain_todo": {
-    "stmt2": true
-  }
-}
+{}