Skip to content
83 changes: 83 additions & 0 deletions eng/tools/azure-sdk-tools/packaging_tools/generate_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,89 @@
DEFAULT_DEST_FOLDER = "./dist"
_DPG_README = "README.md"

# ---------------------------------------------------------------------------
# Sanitize invalid Python escape sequences in generated docstrings.
#
# Python generators (TypeSpec and Swagger) write `@doc` strings verbatim into
# generated source. When a doc contains a backslash followed by a character
# that is not a valid Python escape (e.g. ``\W`` in ``(Regex match [\W_])``),
# Python 3.12+ emits a SyntaxWarning on import (and will become SyntaxError in
# a future version).
#
# See:
# - https://github.com/Azure/azure-sdk-for-python/issues/47011
# - https://github.com/microsoft/typespec/issues/10784
#
# This sanitizer escapes those backslashes inside triple-quoted string literals
# of just-generated files only. It is idempotent: once ``\X`` is rewritten to
# ``\\X`` it no longer matches.
# ---------------------------------------------------------------------------

# Match a triple-quoted string literal at a token boundary, capturing the
# full Python string prefix. Per PEP, valid Python 3 string prefixes are:
# (empty), u, U, b, B, f, F, r, R, br, bR, Br, BR, rb, rB, Rb, RB,
# fr, fR, Fr, FR, rf, rF, Rf, RF. We anchor with ``(?<!\w)`` so the prefix
# can't start mid-identifier (e.g. ``foo r"""..."""`` is still matched, but
# ``barf"""..."""`` is not — though that wouldn't be valid Python anyway).
_TRIPLE_QUOTED_RE = re.compile(
r"(?<!\w)"
r"(?P<prefix>(?:[bB][rR]?|[rR][bBfF]?|[fF][rR]?|[uU])?)"
r'(?P<quote>"""|\'\'\')'
r"(?P<body>.*?)"
r"(?P=quote)",
re.DOTALL,
)
Comment thread
msyyc marked this conversation as resolved.
# Process one escape at a time. The first alternative consumes a *valid*
# escape pair as a single unit (so we never re-process its second char),
# making the overall transformation idempotent. The second alternative
# matches any remaining backslash + char and is the one we rewrite.
_ESCAPE_PROCESSOR = re.compile(
r'\\(?P<valid>\\|[\'"abfnrtv0-7xNuU\n])|\\(?P<invalid>.)',
re.DOTALL,
)


def _process_escape(match: "re.Match") -> str:
if match.group("invalid") is not None:
return "\\\\" + match.group("invalid")
return match.group(0)


def _fix_triple_quoted(match: "re.Match") -> str:
prefix = match.group("prefix")
# Raw (r/R) and byte (b/B) string literals must be left untouched.
if prefix and ("r" in prefix.lower() or "b" in prefix.lower()):
return match.group(0)
body = match.group("body")
fixed = _ESCAPE_PROCESSOR.sub(_process_escape, body)
if fixed == body:
return match.group(0)
return f"{prefix}{match.group('quote')}{fixed}{match.group('quote')}"


def sanitize_generated_docstrings(sdk_code_path: str) -> None:
"""Escape invalid Python escape sequences in generated triple-quoted docstrings.
Comment thread
msyyc marked this conversation as resolved.

Workaround for microsoft/typespec#10784 (and the swagger generator
equivalent). Scans ``<sdk_code_path>/azure/**/*.py`` (excluding
``*_patch.py``) and rewrites only triple-quoted string literals.
Idempotent.
"""
Comment thread
msyyc marked this conversation as resolved.
pkg_root = Path(sdk_code_path) / "azure"
if not pkg_root.exists():
return
for py_file in pkg_root.rglob("*.py"):
if py_file.name.endswith("_patch.py"):
continue
try:
original = py_file.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError):
continue
fixed = _TRIPLE_QUOTED_RE.sub(_fix_triple_quoted, original)
if fixed != original:
py_file.write_text(fixed, encoding="utf-8")
_LOGGER.info(f"sanitized invalid escape sequences in {py_file}")
Comment thread
msyyc marked this conversation as resolved.


# tsp example: "../azure-rest-api-specs/specification/informatica/Informatica.DataManagement"
def del_outdated_generated_files(tsp: str):
Expand Down
9 changes: 9 additions & 0 deletions eng/tools/azure-sdk-tools/packaging_tools/sdk_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
dpg_relative_folder,
gen_typespec,
del_outdated_generated_files,
sanitize_generated_docstrings,
)
from .package_utils import create_package, check_file
from .sdk_changelog import main as sdk_changelog_generate
Expand Down Expand Up @@ -179,6 +180,14 @@ def main(generate_input, generate_output):
try:
package_total.add(package_name)
sdk_code_path = str(Path(sdk_folder, folder_name, package_name))
# Sanitize invalid Python escape sequences (e.g. `\W`) in
# generated docstrings to avoid SyntaxWarning on Python 3.12+.
# See https://github.com/Azure/azure-sdk-for-python/issues/47011
# and https://github.com/microsoft/typespec/issues/10784.
try:
sanitize_generated_docstrings(sdk_code_path)
except Exception as e:
_LOGGER.warning(f"Fail to sanitize generated docstrings for {package_name} in {readme_or_tsp}: {e}")
if package_name not in result:
package_entry = {}
package_entry["packageName"] = package_name
Expand Down
143 changes: 143 additions & 0 deletions eng/tools/azure-sdk-tools/tests/test_sanitize_docstrings.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
from packaging_tools.generate_utils import (
_TRIPLE_QUOTED_RE,
_fix_triple_quoted,
sanitize_generated_docstrings,
)


def _run(src: str) -> str:
return _TRIPLE_QUOTED_RE.sub(_fix_triple_quoted, src)


def test_fixes_invalid_W_escape():
src = '"""hi (Regex match [\\W_])"""'
assert _run(src) == '"""hi (Regex match [\\\\W_])"""'


def test_fixes_other_invalid_escapes():
# \d \s are also invalid Python escapes
src = '"""\\d \\s \\p"""'
assert _run(src) == '"""\\\\d \\\\s \\\\p"""'


def test_idempotent():
once = _run('"""[\\W_]"""')
assert _run(once) == once


def test_leaves_valid_escapes_alone():
src = '"""line1\\nline2\\ttab \\\\backslash \\"quote\\""""'
assert _run(src) == src


def test_leaves_raw_strings_alone():
src = 'r"""[\\W_]"""'
assert _run(src) == src

Comment thread
msyyc marked this conversation as resolved.

def test_leaves_byte_strings_alone():
# Byte triple-quoted strings are rare in docstrings but must not be touched.
# The regex matches them, but `_fix_triple_quoted` bails out because the
# captured prefix contains `b`.
src = 'b"""[\\W_]"""'
Comment thread
msyyc marked this conversation as resolved.
assert _run(src) == src


def test_leaves_raw_fstring_alone_rf_prefix():
# Regression: ``rf"""..."""`` was previously sanitized because the
# lookbehind only saw ``f`` immediately before the quote.
src = 'rf"""[\\W_]"""'
assert _run(src) == src


def test_leaves_raw_fstring_alone_fr_prefix():
src = 'fr"""[\\W_]"""'
assert _run(src) == src


def test_leaves_raw_byte_string_alone_rb_prefix():
src = 'rb"""[\\W_]"""'
assert _run(src) == src


def test_leaves_raw_byte_string_alone_br_prefix():
src = 'br"""[\\W_]"""'
assert _run(src) == src


def test_leaves_raw_byte_string_alone_mixed_case_prefix():
for src in ('Rb"""[\\W_]"""', 'bR"""[\\W_]"""', 'rF"""[\\W_]"""', 'Fr"""[\\W_]"""'):
assert _run(src) == src


def test_leaves_single_quoted_strings_alone():
# Generated docstrings are always triple-quoted; regex literals in
# generated code use double-quoted strings and must not be modified.
src = 'PATTERN = "\\W"'
assert _run(src) == src


def test_single_triple_quoted_string():
src = "'''(Regex match [\\W_])'''"
assert _run(src) == "'''(Regex match [\\\\W_])'''"


def test_unicode_prefix_preserved():
src = 'u"""[\\W_]"""'
assert _run(src) == 'u"""[\\\\W_]"""'


def test_fstring_prefix_preserved():
src = 'f"""[\\W_]"""'
assert _run(src) == 'f"""[\\\\W_]"""'


def test_multiline_docstring():
src = '"""line1\nHas a special character (Regex match [\\W_])\nline3"""'
expected = '"""line1\nHas a special character (Regex match [\\\\W_])\nline3"""'
assert _run(src) == expected


def test_sanitize_writes_only_changed_files(tmp_path):
# Simulate an sdk package layout: <pkg>/azure/<ns>/_models.py + _patch.py
pkg = tmp_path / "azure-mgmt-foo"
ns = pkg / "azure" / "mgmt" / "foo"
ns.mkdir(parents=True)
models = ns / "_models.py"
models.write_text(
'class M:\n """Has [\\W_]"""\n pass\n',
encoding="utf-8",
)
# _patch.py is hand-written; must never be touched even if it has invalid escapes
patch = ns / "_patch.py"
patch_src = '"""user-authored [\\W_]"""\n'
patch.write_text(patch_src, encoding="utf-8")
unchanged = ns / "_clean.py"
unchanged_src = '"""nothing to fix here"""\n'
unchanged.write_text(unchanged_src, encoding="utf-8")

sanitize_generated_docstrings(str(pkg))

assert models.read_text(encoding="utf-8") == 'class M:\n """Has [\\\\W_]"""\n pass\n'
assert patch.read_text(encoding="utf-8") == patch_src
assert unchanged.read_text(encoding="utf-8") == unchanged_src


def test_sanitize_is_idempotent_on_disk(tmp_path):
pkg = tmp_path / "azure-mgmt-foo"
ns = pkg / "azure" / "mgmt" / "foo"
ns.mkdir(parents=True)
models = ns / "_models.py"
models.write_text('"""Has [\\W_]"""\n', encoding="utf-8")

sanitize_generated_docstrings(str(pkg))
first = models.read_text(encoding="utf-8")
sanitize_generated_docstrings(str(pkg))
assert models.read_text(encoding="utf-8") == first


def test_sanitize_missing_azure_dir_is_noop(tmp_path):
# No azure/ subdir -> nothing to do, no crash.
pkg = tmp_path / "azure-mgmt-foo"
pkg.mkdir()
sanitize_generated_docstrings(str(pkg))
Loading