diff --git a/eng/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/eng/tools/azure-sdk-tools/packaging_tools/generate_utils.py index d88e2f79164e..f5dd14db55b6 100644 --- a/eng/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/eng/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -39,6 +39,89 @@ DEFAULT_DEST_FOLDER = "./dist" _DPG_README = "README.md" +# --------------------------------------------------------------------------- +# Sanitize invalid Python escape sequences in generated docstrings. +# +# Python generators (TypeSpec and Swagger) write `@doc` strings verbatim into +# generated source. When a doc contains a backslash followed by a character +# that is not a valid Python escape (e.g. ``\W`` in ``(Regex match [\W_])``), +# Python 3.12+ emits a SyntaxWarning on import (and will become SyntaxError in +# a future version). +# +# See: +# - https://github.com/Azure/azure-sdk-for-python/issues/47011 +# - https://github.com/microsoft/typespec/issues/10784 +# +# This sanitizer escapes those backslashes inside triple-quoted string literals +# of just-generated files only. It is idempotent: once ``\X`` is rewritten to +# ``\\X`` it no longer matches. +# --------------------------------------------------------------------------- + +# Match a triple-quoted string literal at a token boundary, capturing the +# full Python string prefix. Per PEP, valid Python 3 string prefixes are: +# (empty), u, U, b, B, f, F, r, R, br, bR, Br, BR, rb, rB, Rb, RB, +# fr, fR, Fr, FR, rf, rF, Rf, RF. We anchor with ``(?(?:[bB][rR]?|[rR][bBfF]?|[fF][rR]?|[uU])?)" + r'(?P"""|\'\'\')' + r"(?P.*?)" + r"(?P=quote)", + re.DOTALL, +) +# Process one escape at a time. The first alternative consumes a *valid* +# escape pair as a single unit (so we never re-process its second char), +# making the overall transformation idempotent. The second alternative +# matches any remaining backslash + char and is the one we rewrite. +_ESCAPE_PROCESSOR = re.compile( + r'\\(?P\\|[\'"abfnrtv0-7xNuU\n])|\\(?P.)', + re.DOTALL, +) + + +def _process_escape(match: "re.Match") -> str: + if match.group("invalid") is not None: + return "\\\\" + match.group("invalid") + return match.group(0) + + +def _fix_triple_quoted(match: "re.Match") -> str: + prefix = match.group("prefix") + # Raw (r/R) and byte (b/B) string literals must be left untouched. + if prefix and ("r" in prefix.lower() or "b" in prefix.lower()): + return match.group(0) + body = match.group("body") + fixed = _ESCAPE_PROCESSOR.sub(_process_escape, body) + if fixed == body: + return match.group(0) + return f"{prefix}{match.group('quote')}{fixed}{match.group('quote')}" + + +def sanitize_generated_docstrings(sdk_code_path: str) -> None: + """Escape invalid Python escape sequences in generated triple-quoted docstrings. + + Workaround for microsoft/typespec#10784 (and the swagger generator + equivalent). Scans ``/azure/**/*.py`` (excluding + ``*_patch.py``) and rewrites only triple-quoted string literals. + Idempotent. + """ + pkg_root = Path(sdk_code_path) / "azure" + if not pkg_root.exists(): + return + for py_file in pkg_root.rglob("*.py"): + if py_file.name.endswith("_patch.py"): + continue + try: + original = py_file.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + fixed = _TRIPLE_QUOTED_RE.sub(_fix_triple_quoted, original) + if fixed != original: + py_file.write_text(fixed, encoding="utf-8") + _LOGGER.info(f"sanitized invalid escape sequences in {py_file}") + # tsp example: "../azure-rest-api-specs/specification/informatica/Informatica.DataManagement" def del_outdated_generated_files(tsp: str): diff --git a/eng/tools/azure-sdk-tools/packaging_tools/sdk_generator.py b/eng/tools/azure-sdk-tools/packaging_tools/sdk_generator.py index abfad0733554..e9624a5db813 100644 --- a/eng/tools/azure-sdk-tools/packaging_tools/sdk_generator.py +++ b/eng/tools/azure-sdk-tools/packaging_tools/sdk_generator.py @@ -32,6 +32,7 @@ dpg_relative_folder, gen_typespec, del_outdated_generated_files, + sanitize_generated_docstrings, ) from .package_utils import create_package, check_file from .sdk_changelog import main as sdk_changelog_generate @@ -179,6 +180,14 @@ def main(generate_input, generate_output): try: package_total.add(package_name) sdk_code_path = str(Path(sdk_folder, folder_name, package_name)) + # Sanitize invalid Python escape sequences (e.g. `\W`) in + # generated docstrings to avoid SyntaxWarning on Python 3.12+. + # See https://github.com/Azure/azure-sdk-for-python/issues/47011 + # and https://github.com/microsoft/typespec/issues/10784. + try: + sanitize_generated_docstrings(sdk_code_path) + except Exception as e: + _LOGGER.warning(f"Fail to sanitize generated docstrings for {package_name} in {readme_or_tsp}: {e}") if package_name not in result: package_entry = {} package_entry["packageName"] = package_name diff --git a/eng/tools/azure-sdk-tools/tests/test_sanitize_docstrings.py b/eng/tools/azure-sdk-tools/tests/test_sanitize_docstrings.py new file mode 100644 index 000000000000..8f2c492ddd6f --- /dev/null +++ b/eng/tools/azure-sdk-tools/tests/test_sanitize_docstrings.py @@ -0,0 +1,143 @@ +from packaging_tools.generate_utils import ( + _TRIPLE_QUOTED_RE, + _fix_triple_quoted, + sanitize_generated_docstrings, +) + + +def _run(src: str) -> str: + return _TRIPLE_QUOTED_RE.sub(_fix_triple_quoted, src) + + +def test_fixes_invalid_W_escape(): + src = '"""hi (Regex match [\\W_])"""' + assert _run(src) == '"""hi (Regex match [\\\\W_])"""' + + +def test_fixes_other_invalid_escapes(): + # \d \s are also invalid Python escapes + src = '"""\\d \\s \\p"""' + assert _run(src) == '"""\\\\d \\\\s \\\\p"""' + + +def test_idempotent(): + once = _run('"""[\\W_]"""') + assert _run(once) == once + + +def test_leaves_valid_escapes_alone(): + src = '"""line1\\nline2\\ttab \\\\backslash \\"quote\\""""' + assert _run(src) == src + + +def test_leaves_raw_strings_alone(): + src = 'r"""[\\W_]"""' + assert _run(src) == src + + +def test_leaves_byte_strings_alone(): + # Byte triple-quoted strings are rare in docstrings but must not be touched. + # The regex matches them, but `_fix_triple_quoted` bails out because the + # captured prefix contains `b`. + src = 'b"""[\\W_]"""' + assert _run(src) == src + + +def test_leaves_raw_fstring_alone_rf_prefix(): + # Regression: ``rf"""..."""`` was previously sanitized because the + # lookbehind only saw ``f`` immediately before the quote. + src = 'rf"""[\\W_]"""' + assert _run(src) == src + + +def test_leaves_raw_fstring_alone_fr_prefix(): + src = 'fr"""[\\W_]"""' + assert _run(src) == src + + +def test_leaves_raw_byte_string_alone_rb_prefix(): + src = 'rb"""[\\W_]"""' + assert _run(src) == src + + +def test_leaves_raw_byte_string_alone_br_prefix(): + src = 'br"""[\\W_]"""' + assert _run(src) == src + + +def test_leaves_raw_byte_string_alone_mixed_case_prefix(): + for src in ('Rb"""[\\W_]"""', 'bR"""[\\W_]"""', 'rF"""[\\W_]"""', 'Fr"""[\\W_]"""'): + assert _run(src) == src + + +def test_leaves_single_quoted_strings_alone(): + # Generated docstrings are always triple-quoted; regex literals in + # generated code use double-quoted strings and must not be modified. + src = 'PATTERN = "\\W"' + assert _run(src) == src + + +def test_single_triple_quoted_string(): + src = "'''(Regex match [\\W_])'''" + assert _run(src) == "'''(Regex match [\\\\W_])'''" + + +def test_unicode_prefix_preserved(): + src = 'u"""[\\W_]"""' + assert _run(src) == 'u"""[\\\\W_]"""' + + +def test_fstring_prefix_preserved(): + src = 'f"""[\\W_]"""' + assert _run(src) == 'f"""[\\\\W_]"""' + + +def test_multiline_docstring(): + src = '"""line1\nHas a special character (Regex match [\\W_])\nline3"""' + expected = '"""line1\nHas a special character (Regex match [\\\\W_])\nline3"""' + assert _run(src) == expected + + +def test_sanitize_writes_only_changed_files(tmp_path): + # Simulate an sdk package layout: /azure//_models.py + _patch.py + pkg = tmp_path / "azure-mgmt-foo" + ns = pkg / "azure" / "mgmt" / "foo" + ns.mkdir(parents=True) + models = ns / "_models.py" + models.write_text( + 'class M:\n """Has [\\W_]"""\n pass\n', + encoding="utf-8", + ) + # _patch.py is hand-written; must never be touched even if it has invalid escapes + patch = ns / "_patch.py" + patch_src = '"""user-authored [\\W_]"""\n' + patch.write_text(patch_src, encoding="utf-8") + unchanged = ns / "_clean.py" + unchanged_src = '"""nothing to fix here"""\n' + unchanged.write_text(unchanged_src, encoding="utf-8") + + sanitize_generated_docstrings(str(pkg)) + + assert models.read_text(encoding="utf-8") == 'class M:\n """Has [\\\\W_]"""\n pass\n' + assert patch.read_text(encoding="utf-8") == patch_src + assert unchanged.read_text(encoding="utf-8") == unchanged_src + + +def test_sanitize_is_idempotent_on_disk(tmp_path): + pkg = tmp_path / "azure-mgmt-foo" + ns = pkg / "azure" / "mgmt" / "foo" + ns.mkdir(parents=True) + models = ns / "_models.py" + models.write_text('"""Has [\\W_]"""\n', encoding="utf-8") + + sanitize_generated_docstrings(str(pkg)) + first = models.read_text(encoding="utf-8") + sanitize_generated_docstrings(str(pkg)) + assert models.read_text(encoding="utf-8") == first + + +def test_sanitize_missing_azure_dir_is_noop(tmp_path): + # No azure/ subdir -> nothing to do, no crash. + pkg = tmp_path / "azure-mgmt-foo" + pkg.mkdir() + sanitize_generated_docstrings(str(pkg))