From 90b3c32898fa7f0892674994d92dcc019637f583 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 7 Jul 2026 15:51:11 -0700 Subject: [PATCH 1/2] docs: describe the parser as vocabulary + positional layers in README Reorganize the README's how-it-works prose around the two-layer model: a vocabulary layer that recognizes pieces by what they are (the customizable sets), and a positional layer that assigns the rest by where it sits. States explicitly that parsing is deterministic (no statistical models) and that the positional layer assumes Western name order. All claims from the previous prose are preserved. Co-Authored-By: Claude Fable 5 --- README.rst | 58 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index bbecaba..0a09a80 100644 --- a/README.rst +++ b/README.rst @@ -29,19 +29,35 @@ are optional. Comma-separated format like "Last, First" is also supported. 2. Lastname [Suffix], Title Firstname (Nickname) Middle Middle[,] Suffix [, Suffix] 3. Title Firstname M Lastname [Suffix], Suffix [Suffix] [, Suffix] -Instantiating the `HumanName` class with a string splits on commas and then spaces, -classifying name parts based on placement in the string and matches against known name -pieces like titles and suffixes. - -It correctly handles some common conjunctions and special prefixes to last names -like "del". Titles and conjunctions can be chained together to handle complex -titles like "Asst Secretary of State". It can also try to correct capitalization -of names that are all upper- or lowercase names. - -It attempts the best guess that can be made with a simple, rule-based approach. -Its main use case is English and it is not likely to be useful for languages -that do not conform to the supported name structure. It's not perfect, but it -gets you pretty far. +How It Works +~~~~~~~~~~~~ + +The parser works in two layers. + +A **vocabulary layer** recognizes name pieces by what they are, using +configurable sets of known words: titles ("Dr."), suffixes ("III", "PhD"), +last-name prefixes ("de la"), conjunctions ("y", "&"), and delimited +nicknames ("Doc"). Titles and conjunctions chain together to handle complex +titles like "Asst Secretary of State"; prefixes join forward so "de la Vega" +stays one last name. This layer doesn't care where in the string a word +appears — and it's the layer you customize, by adding or removing entries +in the sets to fit your dataset. + +A **positional layer** then assigns everything the vocabulary layer didn't +claim, based purely on where it sits: the first unclaimed word is the first +name, the last one is the last name, and anything between them is a middle +name. There is no semantic understanding — "Dr" is a title when it comes +before a name and a suffix when it comes after ("pre-nominal" and +"post-nominal" would probably be better names) — and no attempt to correct +mistakes in the input. + +It attempts the best guess that can be made with a simple, deterministic, +rule-based approach — no statistical models or machine learning; the same +input always parses the same way. The positional layer assumes Western name +order (given name first), so the main use case is English and other +languages that share that structure. It can also try to correct the +capitalization of names that are all upper- or lowercase. It's not perfect, +but it gets you pretty far. Installation ------------ @@ -88,11 +104,8 @@ Quick Start Example 'Juan de la Vega' -The parser does not attempt to correct mistakes in the input. It mostly just splits on white -space and puts things in buckets based on their position in the string. This also means -the difference between 'title' and 'suffix' is positional, not semantic. "Dr" is a title -when it comes before the name and a suffix when it comes after. ("Pre-nominal" -and "post-nominal" would probably be better names.) +Because the positional layer has no semantic understanding, position is +everything: :: @@ -111,10 +124,11 @@ and "post-nominal" would probably be better names.) Customization ------------- -Your project may need some adjustment for your dataset. You can -do this in your own pre- or post-processing, by `customizing the configured pre-defined -sets`_ of titles, prefixes, etc., or by subclassing the `HumanName` class. See the -`full documentation`_ for more information. +Your project may need some adjustment for your dataset. Most customization +is vocabulary — `customizing the configured pre-defined sets`_ of titles, +prefixes, etc. that the vocabulary layer matches against. You can also do +your own pre- or post-processing, or subclass the `HumanName` class for +deeper changes. See the `full documentation`_ for more information. `Full documentation`_ From 9a0115318caac0236b93ae72ed9824038ec7ef7e Mon Sep 17 00:00:00 2001 From: apoorva-01 Date: Wed, 8 Jul 2026 06:11:03 +0530 Subject: [PATCH 2/2] Strip invisible bidi control characters during preprocessing These marks (U+200F and friends) ride along with copy-pasted RTL names and render as nothing, but they glue onto first/last so the parsed parts quietly stop comparing equal to the clean string. Stripped first, before the rest of preprocessing, so a mark sitting inside a token can't defeat the Ph.D. and nickname matching. --- docs/customize.rst | 18 ++++++++++++++++++ docs/release_log.rst | 4 ++++ nameparser/config/regexes.py | 7 +++++++ nameparser/parser.py | 14 +++++++++++++- tests/test_output_format.py | 24 ++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 1 deletion(-) diff --git a/docs/customize.rst b/docs/customize.rst index cb2be8c..7397c68 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -501,6 +501,24 @@ You can turn this off by setting the ``emoji`` regex to ``False``. >>> str(hn) 'Sam 😊 Smith' +Don't Remove Bidi Control Characters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, invisible bidirectional control characters (the left-to-right and +right-to-left marks and friends, common in copy-pasted right-to-left names) are +removed from the input string before the name is parsed. You can turn this off +by setting the ``bidi`` regex to ``False``. + +.. doctest:: + + >>> from nameparser import HumanName + >>> from nameparser.config import Constants + >>> constants = Constants() + >>> constants.regexes.bidi = False + >>> hn = HumanName("\u200fJohn\u200f Smith", constants=constants) + >>> hn.first == "\u200fJohn\u200f" + True + Config Changes May Need Parse Refresh ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/release_log.rst b/docs/release_log.rst index ab17c53..1c5ccca 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -1,5 +1,9 @@ Release Log =========== +* 1.3.1 - Unreleased + + - Fix invisible Unicode bidirectional control characters (LRM/RLM/ALM, the embedding/override marks, and the isolates U+2066–U+2069) surviving parsing and sticking to ``first``/``last``/etc., so a copy-pasted right-to-left name silently failed equality and dedup. They are now stripped in preprocessing like emoji; disable via ``CONSTANTS.regexes.bidi = False`` (closes #266) + * 1.3.0 - July 5, 2026 **Breaking Changes & Deprecations** diff --git a/nameparser/config/regexes.py b/nameparser/config/regexes.py index 9c496a0..6ff0b6b 100644 --- a/nameparser/config/regexes.py +++ b/nameparser/config/regexes.py @@ -6,6 +6,12 @@ '\U0001F680-\U0001F6FF' '\u2600-\u26FF\u2700-\u27BF]+') +# Invisible bidirectional formatting characters: ALM, LRM, RLM, the +# embedding/override marks (LRE/RLE/PDF/LRO/RLO) and the isolates +# (LRI/RLI/FSI/PDI). Copy-pasted RTL names often carry these; they render +# as nothing but stick to name parts and break equality/lookups. +re_bidi = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+') + EMPTY_REGEX = re.compile('') REGEXES = { @@ -20,6 +26,7 @@ "no_vowels": re.compile(r'^[^aeyiuo]+$', re.I), "period_not_at_end": re.compile(r'.*\..+$', re.I), "emoji": re_emoji, + "bidi": re_bidi, "phd": re.compile(r'\s(ph\.?\s+d\.?)', re.I), "space_before_comma": re.compile(r'\s+,'), "east_slavic_patronymic": re.compile( diff --git a/nameparser/parser.py b/nameparser/parser.py index 1fd8fd0..c649e7c 100644 --- a/nameparser/parser.py +++ b/nameparser/parser.py @@ -928,9 +928,11 @@ def pre_process(self) -> None: This method happens at the beginning of the :py:func:`parse_full_name` before any other processing of the string aside from unicode normalization, so it's a good place to do any custom handling in a - subclass. Runs :py:func:`parse_nicknames` and :py:func:`squash_emoji`. + subclass. Runs :py:func:`squash_bidi`, :py:func:`parse_nicknames` and + :py:func:`squash_emoji`. """ + self.squash_bidi() self.fix_phd() self.parse_nicknames() self.squash_emoji() @@ -1133,6 +1135,16 @@ def squash_emoji(self) -> None: if re_emoji and re_emoji.search(self._full_name): self._full_name = re_emoji.sub('', self._full_name) + def squash_bidi(self) -> None: + """ + Remove invisible bidirectional control characters from the input + string. They carry no name content but stick to the parts they + surround, so parsed attributes stop comparing equal to the clean name. + """ + re_bidi = self.C.regexes.bidi + if re_bidi and re_bidi.search(self._full_name): + self._full_name = re_bidi.sub('', self._full_name) + def handle_firstnames(self) -> None: """ If there are only two parts and one is a title, assume it's a last name diff --git a/tests/test_output_format.py b/tests/test_output_format.py index 2fb48f6..1336410 100644 --- a/tests/test_output_format.py +++ b/tests/test_output_format.py @@ -136,3 +136,27 @@ def test_keep_emojis(self) -> None: self.m(hn.last, "Smith😊", hn) self.assertEqual(str(hn), "∫≜⩕ Smith😊") # test cleanup + + def test_remove_bidi_control_chars(self) -> None: + # LRM/RLM and friends ride along with copy-pasted names and stick to + # the parts they surround. Covers every character in the bidi set. + for mark in ("\u200e", "\u200f", "\u061c", "\u202a", "\u202b", + "\u202c", "\u202d", "\u202e", "\u2066", "\u2067", + "\u2068", "\u2069"): + hn = HumanName(mark + "John" + mark + " Smith") + self.m(hn.first, "John", hn) + self.m(hn.last, "Smith", hn) + + def test_bidi_stripped_name_compares_equal(self) -> None: + # The reported symptom: an invisible RLM around an RTL name makes the + # parsed part fail equality against the clean string (issue #266). + hn = HumanName("\u200fمحمد بن سلمان\u200f") + self.assertEqual(hn.first, "محمد") + + def test_keep_bidi_control_chars(self) -> None: + from nameparser.config import Constants + constants = Constants() + constants.regexes.bidi = False # type: ignore[assignment] + hn = HumanName("\u200fJohn\u200f Smith", constants) + self.m(hn.first, "\u200fJohn\u200f", hn) + self.m(hn.last, "Smith", hn)