diff --git a/backend/src/chitai/services/matching.py b/backend/src/chitai/services/matching.py new file mode 100644 index 0000000..3dd9bd5 --- /dev/null +++ b/backend/src/chitai/services/matching.py @@ -0,0 +1,163 @@ +# src/chitai/services/matching.py + +""" +Normalisation for book-level duplicate detection. + +Two copies of one book rarely agree on how it is written down. One says +`The Metamorphosis`, the other `Metamorphosis`; one credits `Kafka, Franz`, the other +`Franz Kafka`; one carries the ISBN-10 and the other the ISBN-13 of the same edition. +These functions reduce each of those to a single key, so the comparison is an equality +test the database can index rather than a similarity score nobody can explain. + +Everything here is pure: the keys are computed once and stored on the row (see +`Book.normalized_title`, `Author.normalized_name`, `Identifier.normalized_value`), so +no Postgres extension is needed at query time. +""" + +from __future__ import annotations + +import re +import unicodedata + +from chitai.services.utils import is_valid_isbn, isbn10_to_isbn13 + + +# Asides a title carries that say nothing about which book it is: +# "Frankenstein (Illustrated)", "Dune [Deluxe]". +_BRACKETED = re.compile(r"[(\[{][^)\]}]*[)\]}]") + +# Edition and format qualifiers, matched only as a *trailing* run of words. Anchoring +# to the end is what keeps "The Illustrated Man" a book and "Moby Dick Illustrated" a +# format note — a qualifier trails the title, it is never the thing the title is about. +_EDITION_NOISE = re.compile( + r"\s+(?:" + r"\d+(?:st|nd|rd|th)?\s+ed(?:ition|n)?" + r"|(?:first|second|third|fourth|fifth|sixth|new|revised|expanded|updated|" + r"annotated|illustrated|unabridged|abridged|complete|definitive|deluxe|" + r"anniversary|collectors|international|kindle|paperback|hardcover|hardback|" + r"ebook|audiobook)" + r"(?:\s+(?:and|&)\s+\w+)*" + r"(?:\s+ed(?:ition|n)?)?" + r")$" +) + +_LEADING_ARTICLE = re.compile(r"^(?:the|a|an)\s+") + +# Anything that is not a letter, a digit or a space, once accents are gone. +_PUNCTUATION = re.compile(r"[^0-9a-z ]+") + +_WHITESPACE = re.compile(r"\s+") + +# `J. R. R.` survives punctuation stripping as three one-letter words; `J.R.R.` as one. +# Joining any run of them makes both `jrr`. +_INITIAL_RUN = re.compile(r"\b(?:[a-z] )+[a-z]\b") + +# ISBNs are the same number under several names; everything else keeps its own. +_ISBN_NAMES = {"isbn", "isbn-10", "isbn10", "isbn-13", "isbn13"} + +# Generated fresh for every build of a file, so two copies of one book never share one. +# Matching on them would only re-find files the hash check already catches. +_PER_BUILD_NAMES = {"uuid", "urn:uuid"} + +# Below this an identifier is not specific enough to be evidence: a Calibre `id` of +# "42" would otherwise pair two unrelated books. +_MIN_IDENTIFIER_LENGTH = 4 + + +def _fold(text: str) -> str: + """Casefolded, accent-free, punctuation-free, single-spaced.""" + decomposed = unicodedata.normalize("NFKD", text.casefold()) + unaccented = "".join(c for c in decomposed if not unicodedata.combining(c)) + + return _WHITESPACE.sub(" ", _PUNCTUATION.sub(" ", unaccented)).strip() + + +def normalize_title(title: str | None) -> str: + """ + Reduce a title to the key two copies of one book should share. + + `Book.subtitle` is already split off by `Extractor.format_book_title`, so only what + is left in the title column is considered here. + + Args: + title: The title as it was stored. + + Returns: + The comparison key, or an empty string if nothing survives normalisation — + which is the signal not to match on the title at all. + """ + if not title: + return "" + + folded = _fold(_BRACKETED.sub(" ", title).replace("&", " and ")) + + # Repeated because qualifiers stack: "Dune Deluxe Edition Illustrated". + while (trimmed := _EDITION_NOISE.sub("", folded)) != folded: + folded = trimmed + + # An article says nothing, but a title that is only an article is not improved by + # having none, and neither is one that noise removal emptied out. + return _LEADING_ARTICLE.sub("", folded, count=1) or folded + + +def normalize_author(name: str | None) -> str: + """ + Reduce an author's name to the key their other books should share. + + Deliberately not reduced to surname plus initial: that collides unrelated people, + and a wrong match here is a book pointed at a stranger's shelf. + + Args: + name: The name as it was stored, in either `Franz Kafka` or `Kafka, Franz` form. + + Returns: + The comparison key, or an empty string if nothing survives normalisation. + """ + if not name: + return "" + + # `Kafka, Franz` is one name written backwards. More than one comma is a list, or a + # suffix, and guessing at either does more harm than leaving it alone. + if name.count(",") == 1: + surname, forename = name.split(",") + name = f"{forename.strip()} {surname.strip()}" + + folded = _fold(name) + + return _INITIAL_RUN.sub(lambda run: run.group().replace(" ", ""), folded) + + +def normalize_identifier(name: str, value: str) -> str | None: + """ + Reduce one identifier to a `scheme:value` key, if it can carry a match at all. + + Args: + name: What kind of identifier it is, as stored. + value: The identifier itself. + + Returns: + The key, or None when the identifier is no use for matching: a per-build UUID, + something too short to be evidence, or an ISBN that fails its own checksum. + """ + name = (name or "").strip().casefold() + value = (value or "").strip() + + if not name or not value or name in _PER_BUILD_NAMES: + return None + + if name in _ISBN_NAMES: + digits = re.sub(r"[^0-9Xx]", "", value).upper() + + if not is_valid_isbn(digits): + return None + + # One scheme for both forms: a publisher prints whichever it likes, and the + # ISBN-10 and ISBN-13 of an edition are the same number written twice. + isbn = digits if len(digits) == 13 else isbn10_to_isbn13(digits) + return f"isbn:{isbn}" if isbn else None + + folded = _fold(value) or value.casefold() + if len(folded) < _MIN_IDENTIFIER_LENGTH: + return None + + return f"{name}:{folded}" diff --git a/backend/src/chitai/services/utils.py b/backend/src/chitai/services/utils.py index e01c4cc..2702c8f 100644 --- a/backend/src/chitai/services/utils.py +++ b/backend/src/chitai/services/utils.py @@ -512,6 +512,29 @@ def is_valid_isbn10(isbn: str) -> bool: return str(check_digit) == isbn[-1] or (check_digit == 10 and isbn[-1] in "Xx") +def isbn10_to_isbn13(isbn: str) -> str | None: + """ + Convert an ISBN-10 to the ISBN-13 naming the same edition. + + The two are the same number written twice: prefix `978`, drop the ISBN-10 check + digit, recompute the check digit under the ISBN-13 rule. Matching only works if + both forms collapse onto one, since a publisher may print either. + + Args: + isbn: A 10-character ISBN, digits and an optional trailing `X` only. + + Returns: + The equivalent ISBN-13, or None if the input is not a valid ISBN-10. + """ + if not is_valid_isbn(isbn) or len(isbn) != 10: + return None + + digits = f"978{isbn[:9]}" + total = sum(int(digit) * (1 if i % 2 == 0 else 3) for i, digit in enumerate(digits)) + + return f"{digits}{(10 - total % 10) % 10}" + + def is_valid_isbn13(isbn: str) -> bool: """ Validate an ISBN-13 number using its check digit. diff --git a/backend/tests/unit/test_matching.py b/backend/tests/unit/test_matching.py new file mode 100644 index 0000000..1471380 --- /dev/null +++ b/backend/tests/unit/test_matching.py @@ -0,0 +1,125 @@ +"""Tests for the normalization behind book-level duplicate detection.""" + +import pytest + +from chitai.services.matching import ( + normalize_author, + normalize_identifier, + normalize_title, +) +from chitai.services.utils import isbn10_to_isbn13 + + +class TestNormalizeTitle: + """Two copies of one book rarely agree on how the title is written.""" + + @pytest.mark.parametrize( + ("title", "expected"), + [ + ("The Metamorphosis", "metamorphosis"), + ("Metamorphosis", "metamorphosis"), + ("METAMORPHOSIS", "metamorphosis"), + ("A Tale of Two Cities", "tale of two cities"), + ("An Enquiry", "enquiry"), + # Accents, punctuation and ampersands are spelling, not identity. + ("Les Misérables", "les miserables"), + ("Moby Dick; Or, The Whale", "moby dick or the whale"), + ("Sense & Sensibility", "sense and sensibility"), + # Bracketed asides and trailing edition noise say nothing about the book. + ("Frankenstein (Illustrated)", "frankenstein"), + ("Frankenstein [Kindle Edition]", "frankenstein"), + ("Frankenstein, 2nd Edition", "frankenstein"), + ("Frankenstein Revised Edition", "frankenstein"), + ("Dune Deluxe Edition Illustrated", "dune"), + ("", ""), + (None, ""), + ], + ) + def test_titles_that_should_agree(self, title: str | None, expected: str) -> None: + assert normalize_title(title) == expected + + def test_a_qualifier_that_is_the_title_survives(self) -> None: + """A trailing qualifier is noise; the same word at the front is the book.""" + assert normalize_title("The Illustrated Man") == "illustrated man" + + def test_normalization_never_empties_a_title(self) -> None: + """An article-only title is not improved by having no article left.""" + assert normalize_title("The") == "the" + + + +class TestNormalizeAuthor: + """One person, written down several ways.""" + + @pytest.mark.parametrize( + ("name", "expected"), + [ + ("Franz Kafka", "franz kafka"), + ("Kafka, Franz", "franz kafka"), + ("KAFKA, FRANZ", "franz kafka"), + ("Émile Zola", "emile zola"), + ("Doyle, Arthur Conan", "arthur conan doyle"), + # Runs of initials are joined, so spacing them out changes nothing. + ("J.R.R. Tolkien", "jrr tolkien"), + ("J. R. R. Tolkien", "jrr tolkien"), + ("JRR Tolkien", "jrr tolkien"), + ("", ""), + (None, ""), + ], + ) + def test_names_that_should_agree(self, name: str | None, expected: str) -> None: + assert normalize_author(name) == expected + + def test_two_people_are_not_reduced_together(self) -> None: + """Surname plus initial would collide unrelated writers; it is not used.""" + assert normalize_author("Charles Dickens") != normalize_author("Colin Dexter") + + def test_a_list_is_left_alone(self) -> None: + """More than one comma is a list or a suffix, and guessing does more harm.""" + assert normalize_author("Smith, John, Jr.") == "smith john jr" + + +class TestNormalizeIdentifier: + """Identifiers only help if the same edition produces the same key.""" + + def test_isbn_10_and_isbn_13_are_one_key(self) -> None: + assert normalize_identifier("isbn-10", "0486282112") == "isbn:9780486282114" + assert normalize_identifier("isbn-13", "9780486282114") == "isbn:9780486282114" + + @pytest.mark.parametrize( + "written", ["978-0-486-28211-4", "978 0 486 28211 4", "9780486282114"] + ) + def test_formatting_is_not_part_of_an_isbn(self, written: str) -> None: + assert normalize_identifier("isbn", written) == "isbn:9780486282114" + + def test_an_isbn_that_fails_its_checksum_is_no_evidence(self) -> None: + assert normalize_identifier("isbn-13", "9780486282115") is None + + def test_uuids_are_refused(self) -> None: + """Generated per build, so they only re-find what the hash check catches.""" + assert normalize_identifier("uuid", "3f2b1c4e-1111-2222-3333-444455556666") is None + assert normalize_identifier("urn:uuid", "3f2b1c4e-1111-2222-3333-444455556666") is None + + def test_other_schemes_keep_their_own_key(self) -> None: + assert normalize_identifier("asin", "B000FC0PDA") == "asin:b000fc0pda" + assert normalize_identifier("ASIN", "b000fc0pda") == "asin:b000fc0pda" + + def test_something_too_short_is_not_evidence(self) -> None: + """A Calibre id of "42" would otherwise pair two unrelated books.""" + assert normalize_identifier("calibre", "42") is None + + @pytest.mark.parametrize(("name", "value"), [("", "1234567"), ("asin", "")]) + def test_half_an_identifier_is_no_identifier(self, name: str, value: str) -> None: + assert normalize_identifier(name, value) is None + + +class TestIsbnConversion: + def test_isbn_10_converts_to_its_isbn_13(self) -> None: + assert isbn10_to_isbn13("0486282112") == "9780486282114" + + def test_a_trailing_x_is_a_digit(self) -> None: + assert isbn10_to_isbn13("043942089X") == "9780439420891" + + @pytest.mark.parametrize("isbn", ["0486282113", "9780486282114", "nonsense"]) + def test_anything_that_is_not_an_isbn_10_converts_to_nothing(self, isbn: str) -> None: + assert isbn10_to_isbn13(isbn) is None