diff --git a/docs/release_log.rst b/docs/release_log.rst index 6404a12..b34b4c7 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -9,6 +9,7 @@ Release Log - Add ``Lexicon.honorific_tails``, the vocabulary the glued-honorific peel matches: entries that may be split off the END of a name token, matched longest-first. Deliberately a separate, narrower set than the spaced honorific vocabulary — a glued tail has no token boundary to lean on — and every entry is also a ``suffix_words`` entry, since the peeled piece is claimed by ordinary suffix classification. That subset rule is enforced, so extend both fields in the one call — ``Lexicon.default().add(suffix_words={"ちゃま"}, honorific_tails={"ちゃま"})``; adding to ``honorific_tails`` alone raises ``ValueError`` naming the orphan (#308) - Add ``AmbiguityKind.SEGMENTATION``, reported when a surname split had a vocabulary-supported alternative: ``"남궁민수"`` is 남궁 + 민수 by the compound surname but 남 + 궁민수 by the single-syllable one, and longest-match had to pick. A name with only one possible split decided nothing and reports nothing (#271) - Add the Japanese locale pack ``locales.JA`` and the segmenter factory ``locales.ja_segmenter()``, which together divide an unspaced Japanese name: ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` reads ``山田太郎`` as family ``山田``, given ``太郎``. The two halves are separate because no surname list can do this job — family and given names draw on the same kanji and the reading, not the spelling, decides most divisions — so the pack activates the stage and a third-party divider performs it. ``ja_segmenter()`` wraps `namedivider-python `_, installed with the new ``nameparser[ja]`` extra; the core stays dependency-free, and ``ja_segmenter(gbdt=True)`` selects namedivider's more accurate gradient-boosted model, which downloads its data on first use. With both packs registered, ``locales.available()`` is now ``('ja', 'ru', 'tr_az', 'zh')`` (closes #272) + - Add a construction-time ``UserWarning`` for the segmenterless activation gap: ``parser_for(locales.JA)`` without ``segmenter=`` used to build a parser that behaved exactly like a working one minus the division feature — nothing could ever divide the activated scripts, and nothing said so. Building such a parser now warns, naming the dead scripts and the ``segmenter=locales.ja_segmenter()`` call to pass (with the ``nameparser[ja]`` install hint). Any configured segmenter or covering surname vocabulary silences it, so the default parser and the ``zh`` pack never warn — while a from-scratch lexicon with no hangul surnames now warns under the default policy's hangul activation, with ``Policy(segment_scripts=())`` as the offered deactivation - Add ``Segmentation`` and the ``Segmenter`` type alias to the public API, plus the keyword-only ``Parser(segmenter=...)`` hook they describe: any callable from a token's text to a ``Segmentation`` (the interior offsets to cut at, and a confidence) or ``None`` to decline. It is consulted only for scripts listed in ``Policy.segment_scripts``, and only where the surname vocabulary declined first, so ``parser_for(locales.ZH, locales.JA, segmenter=...)`` composes — a listed Chinese surname wins, the segmenter takes the rest. Read that composition with the zh pack's own warning still attached: vocabulary-first means a Japanese kanji name opening on a listed Chinese surname never reaches the segmenter, so ``高橋一郎`` still splits ``高`` + ``橋一郎`` under the stack exactly as it does under ``locales.ZH`` alone. The two packs are alternatives, one per corpus; stack them only for genuinely mixed data that accepts that trade (#272) - Add the ``Script`` members ``HIRAGANA`` and ``KATAKANA``. Two members rather than one ``KANA`` because the parser treats them differently: hiragana never transcribes a foreign name, while a wholly-katakana name usually is one (#272) diff --git a/docs/usage.rst b/docs/usage.rst index 02b6a1e..36b56f1 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -268,6 +268,14 @@ dependency installed with the ``ja`` extra: Both halves are required, and they do different jobs: the ``ja`` pack activates division for Japanese text, and the segmenter performs it. +Forgetting the segmenter is loud: building the parser emits a +``UserWarning`` naming the scripts that could never divide and the +call to pass, because the misconfigured parser would otherwise behave +exactly like a working one minus the feature. The same check guards +any configuration whose activated scripts nothing can serve — a +from-scratch lexicon with no hangul surnames warns under the default +policy, and ``Policy(segment_scripts=())`` is the deactivation the +message offers. ``ja_segmenter()`` wraps namedivider's ``BasicNameDivider``, which reads data bundled in the installed package; ``ja_segmenter(gbdt=True)`` selects its gradient-boosted divider diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 2e104d1..5c24712 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -18,6 +18,7 @@ from nameparser._pipeline import run from nameparser._pipeline._assemble import assemble from nameparser._pipeline._state import ParseState +from nameparser._pipeline._vocab import _SCRIPT_MATCHERS from nameparser._policy import UNSET, Policy, PolicyPatch, _Unset, apply_patch from nameparser._types import ( FOLDED_TAG, ParsedName, Segmenter, Token, _guarded_getstate, @@ -80,6 +81,47 @@ def __post_init__(self) -> None: if self.segmenter is not None and not callable(self.segmenter): raise TypeError( f"segmenter must be callable or None, got {self.segmenter!r}") + # A configuration gap that used to be silent (#272's API, made + # loud before 2.1.0): segment_scripts can activate a script + # that neither the vocabulary nor a segmenter can ever divide + # -- the JA pack's whole shape, when its segmenter is + # forgotten. The parser then behaves identically to a working + # one minus the feature, which reads as "not working" with no + # signal why. Statically decidable here, so say it here; a + # warning rather than an error because the inert pack is a + # pinned, deliberate property (a JA registration must be safe + # without the extra), and warnings are filterable by the rare + # caller who wants exactly that. + if self.segmenter is None: + uncovered = sorted( + script.value + for script in self.policy.segment_scripts + if not any(_SCRIPT_MATCHERS[script](entry) + for entry in self.lexicon.surnames)) + if uncovered: + names = ", ".join(uncovered) + one = len(uncovered) == 1 + # the ja hint only where a Japanese script is among the + # dead ones -- a hangul-only gap (a from-scratch + # lexicon under the default policy) has different + # remedies, and pointing it at ja_segmenter would be a + # non sequitur + ja_hint = ( + " For Japanese, pass " + "segmenter=locales.ja_segmenter() (install with: " + "pip install 'nameparser[ja]')." + if {"han", "hiragana", "katakana"} & set(uncovered) + else "") + warnings.warn( + f"Policy.segment_scripts activates {names} but the " + f"vocabulary has no surnames in " + f"{'that script' if one else 'those scripts'} " + f"and no segmenter is configured: unspaced names " + f"written in {'it' if one else 'them'} will never " + f"divide. Supply covering surnames, pass a " + f"segmenter, or deactivate with " + f"Policy(segment_scripts=()).{ja_hint}", + UserWarning, stacklevel=3) def __repr__(self) -> str: # composes the two bounded component reprs (spec §2 reprs); the @@ -250,4 +292,15 @@ def parser_for(*locales: Locale, base: Parser | None = None, # a subclass with extra mandatory args would break this rewrap raise type(exc)( f"while applying locale {loc.code!r}: {exc}") from exc - return Parser(lexicon=lexicon, policy=policy, segmenter=segmenter) + # Construction warnings (the segmenterless-activation check in + # Parser.__post_init__) re-emit from THIS frame: its stacklevel is + # sized for direct Parser(...) construction, and through this + # function's extra frame the default single-line rendering would + # point into the library instead of at the caller -- the exact + # call the message tells them to change. + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + built = Parser(lexicon=lexicon, policy=policy, segmenter=segmenter) + for w in caught: + warnings.warn(w.message, stacklevel=2) + return built diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 0b6049f..9b75b05 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -371,7 +371,11 @@ def test_ja_pack_alone_is_inert() -> None: # to segment WITH changes no parse at all. Pure pack data, so this # runs whether or not nameparser[ja] is installed -- and it is the # unsegmented pack's entire observable behavior. - p = parser_for(locales.JA) + # construction now SAYS the pack alone cannot divide anything -- + # the segmenterless-activation warning is the loud half of this + # test's claim, and the inert parses below are the quiet half + with pytest.warns(UserWarning, match="ja_segmenter"): + p = parser_for(locales.JA) for name in _ROTATORS["ja"]: assert p.parse(name).as_dict() == _default_parse(name), name @@ -977,9 +981,18 @@ def test_non_interference_all_packs_combined() -> None: all_rotators = [n for code in sorted(_ROTATORS) if code in _PACKED for n in _ROTATORS[code]] corpus = _default_corpus() + all_rotators - packed = parser_for( - *(locales.get(code) for code in sorted(_PACKS)), - segmenter=locales.ja_segmenter() if _JA_AVAILABLE else None) + if _JA_AVAILABLE: + packed = parser_for( + *(locales.get(code) for code in sorted(_PACKS)), + segmenter=locales.ja_segmenter()) + else: + # without the extra, the stack's hiragana activation is + # unservable -- which construction now SAYS (the segmenterless + # warning, pinned in test_parser.py); expected noise here, the + # gate below is what this test is about + with pytest.warns(UserWarning, match="ja_segmenter"): + packed = parser_for( + *(locales.get(code) for code in sorted(_PACKS))) declared = _assert_non_interference( packed, lambda n: any(m.DEVIATES(n) for m in _PACKS.values()), diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 67b2fac..afe43fe 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -754,3 +754,46 @@ def three(text: str) -> Segmentation | None: n3 = q.parse("Dr 阿明日, Jr.") assert (n3.title, n3.family, n3.given, n3.middle, n3.suffix) == ( "Dr", "阿", "明", "日", "Jr.") + + +def _noop_segmenter(text: str) -> None: + return None + + +def test_segmenterless_activation_without_vocabulary_warns() -> None: + # The JA pack activates HAN and HIRAGANA segmentation but ships no + # vocabulary, and no bundled list could serve those scripts -- so + # without a segmenter, Japanese names can never divide. That is a + # CONFIGURATION gap, not a fact about any name, and it was silent: + # the misconfigured parser behaved identically to a working one + # minus the feature. Statically detectable at construction, so + # warn there. + with pytest.warns(UserWarning, match=r"ja_segmenter"): + parser_for(locales.JA) + + +def test_a_segmenter_silences_the_activation_warning() -> None: + # any segmenter counts: the gap is "nothing can divide these", + # not "you did not use namedivider" + parser_for(locales.JA, segmenter=_noop_segmenter) + + +def test_covered_activation_does_not_warn() -> None: + # HANGUL is served by the census surnames; the zh pack ships the + # vocabulary its own activation needs + Parser() + parser_for(locales.ZH) + + +def test_stacked_activation_warns_only_for_uncovered_scripts() -> None: + # zh covers HAN and the default vocabulary covers HANGUL; only + # HIRAGANA is left unservable, and the message must say WHICH + # scripts are dead rather than naming the whole activation set + with pytest.warns(UserWarning) as caught: + parser_for(locales.ZH, locales.JA) + # select by content: pack application can emit its own warnings + # ahead of construction, so positional indexing is order-fragile + message = next(str(w.message) for w in caught + if "segment_scripts activates" in str(w.message)) + assert "hiragana" in message + assert "hangul" not in message diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 91cbfaa..8b6f4cb 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -7,6 +7,7 @@ against regressions; exploratory fuzzing happened during review. """ import dataclasses +import warnings import pytest from hypothesis import given, settings @@ -452,13 +453,30 @@ def _names_using(draw: st.DrawFn, lexicon: Lexicon, return " ".join(drawn) + + +def _quiet_parser(**kwargs: object) -> Parser: + """Parser construction with the segmenterless-activation warning + ignored: a drawn segment_scripts with a drawn vocabulary that + cannot serve it is exactly the misconfiguration the warning names, + so drawn configs hit it legitimately and constantly. The fuzz here + targets parse behavior, not construction diagnostics -- + tests/v2/test_parser.py pins the warning itself.""" + with warnings.catch_warnings(): + # message-scoped, not a blanket UserWarning ignore: a future + # unrelated construction diagnostic should still fail the fuzz + warnings.filterwarnings( + "ignore", message=r"Policy\.segment_scripts activates") + return Parser(**kwargs) # type: ignore[arg-type] + + @given(_lexicons(), _policies(), st.data()) @settings(max_examples=250, deadline=None, derandomize=True) def test_any_valid_config_still_parses_totally( lexicon: Lexicon, policy: Policy, data: st.DataObject) -> None: # Building the parser is part of the contract: a Lexicon and Policy # that each constructed must also combine. - parser = Parser(lexicon=lexicon, policy=policy) + parser = _quiet_parser(lexicon=lexicon, policy=policy) text = data.draw(_names_using(lexicon, policy)) parsed = parser.parse(text) # must not raise, ever # the anti-#100 invariant, under configuration rather than under @@ -481,8 +499,8 @@ def test_config_values_are_hashable_and_reusable( # safe to build a parser from more than once. assert hash(lexicon) == hash(lexicon) assert {lexicon: 1, policy: 2} - assert Parser(lexicon=lexicon, policy=policy) == Parser( - lexicon=lexicon, policy=policy) + assert _quiet_parser(lexicon=lexicon, policy=policy) == \ + _quiet_parser(lexicon=lexicon, policy=policy) # Values a real caller plausibly passes by mistake: the bare string that @@ -513,7 +531,7 @@ def test_bad_lexicon_field_fails_cleanly(field: str, value: object) -> None: return # Accepted, so it has to survive an actual parse -- construction # succeeding while parsing dies is the same bug one stage later. - Parser(lexicon=lexicon).parse("Dr. John de la Vega III") + _quiet_parser(lexicon=lexicon).parse("Dr. John de la Vega III") @given(st.sampled_from(_POLICY_FIELDS), _HOSTILE) @@ -523,4 +541,4 @@ def test_bad_policy_field_fails_cleanly(field: str, value: object) -> None: policy = Policy(**{field: value}) # type: ignore[arg-type] except (ValueError, TypeError): return - Parser(policy=policy).parse("Dr. John de la Vega III") + _quiet_parser(policy=policy).parse("Dr. John de la Vega III")