feat: add normalisation helpers for book matching
Reduce a title, an author and an identifier to a single comparison key, so two copies of one book can be recognised by equality rather than by a similarity score.
This commit is contained in:
@@ -0,0 +1,163 @@
|
||||
# src/chitai/services/matching.py
|
||||
|
||||
"""
|
||||
Normalisation for book-level duplicate detection.
|
||||
|
||||
Two copies of one book rarely agree on how it is written down. One says
|
||||
`The Metamorphosis`, the other `Metamorphosis`; one credits `Kafka, Franz`, the other
|
||||
`Franz Kafka`; one carries the ISBN-10 and the other the ISBN-13 of the same edition.
|
||||
These functions reduce each of those to a single key, so the comparison is an equality
|
||||
test the database can index rather than a similarity score nobody can explain.
|
||||
|
||||
Everything here is pure: the keys are computed once and stored on the row (see
|
||||
`Book.normalized_title`, `Author.normalized_name`, `Identifier.normalized_value`), so
|
||||
no Postgres extension is needed at query time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from chitai.services.utils import is_valid_isbn, isbn10_to_isbn13
|
||||
|
||||
|
||||
# Asides a title carries that say nothing about which book it is:
|
||||
# "Frankenstein (Illustrated)", "Dune [Deluxe]".
|
||||
_BRACKETED = re.compile(r"[(\[{][^)\]}]*[)\]}]")
|
||||
|
||||
# Edition and format qualifiers, matched only as a *trailing* run of words. Anchoring
|
||||
# to the end is what keeps "The Illustrated Man" a book and "Moby Dick Illustrated" a
|
||||
# format note — a qualifier trails the title, it is never the thing the title is about.
|
||||
_EDITION_NOISE = re.compile(
|
||||
r"\s+(?:"
|
||||
r"\d+(?:st|nd|rd|th)?\s+ed(?:ition|n)?"
|
||||
r"|(?:first|second|third|fourth|fifth|sixth|new|revised|expanded|updated|"
|
||||
r"annotated|illustrated|unabridged|abridged|complete|definitive|deluxe|"
|
||||
r"anniversary|collectors|international|kindle|paperback|hardcover|hardback|"
|
||||
r"ebook|audiobook)"
|
||||
r"(?:\s+(?:and|&)\s+\w+)*"
|
||||
r"(?:\s+ed(?:ition|n)?)?"
|
||||
r")$"
|
||||
)
|
||||
|
||||
_LEADING_ARTICLE = re.compile(r"^(?:the|a|an)\s+")
|
||||
|
||||
# Anything that is not a letter, a digit or a space, once accents are gone.
|
||||
_PUNCTUATION = re.compile(r"[^0-9a-z ]+")
|
||||
|
||||
_WHITESPACE = re.compile(r"\s+")
|
||||
|
||||
# `J. R. R.` survives punctuation stripping as three one-letter words; `J.R.R.` as one.
|
||||
# Joining any run of them makes both `jrr`.
|
||||
_INITIAL_RUN = re.compile(r"\b(?:[a-z] )+[a-z]\b")
|
||||
|
||||
# ISBNs are the same number under several names; everything else keeps its own.
|
||||
_ISBN_NAMES = {"isbn", "isbn-10", "isbn10", "isbn-13", "isbn13"}
|
||||
|
||||
# Generated fresh for every build of a file, so two copies of one book never share one.
|
||||
# Matching on them would only re-find files the hash check already catches.
|
||||
_PER_BUILD_NAMES = {"uuid", "urn:uuid"}
|
||||
|
||||
# Below this an identifier is not specific enough to be evidence: a Calibre `id` of
|
||||
# "42" would otherwise pair two unrelated books.
|
||||
_MIN_IDENTIFIER_LENGTH = 4
|
||||
|
||||
|
||||
def _fold(text: str) -> str:
|
||||
"""Casefolded, accent-free, punctuation-free, single-spaced."""
|
||||
decomposed = unicodedata.normalize("NFKD", text.casefold())
|
||||
unaccented = "".join(c for c in decomposed if not unicodedata.combining(c))
|
||||
|
||||
return _WHITESPACE.sub(" ", _PUNCTUATION.sub(" ", unaccented)).strip()
|
||||
|
||||
|
||||
def normalize_title(title: str | None) -> str:
|
||||
"""
|
||||
Reduce a title to the key two copies of one book should share.
|
||||
|
||||
`Book.subtitle` is already split off by `Extractor.format_book_title`, so only what
|
||||
is left in the title column is considered here.
|
||||
|
||||
Args:
|
||||
title: The title as it was stored.
|
||||
|
||||
Returns:
|
||||
The comparison key, or an empty string if nothing survives normalisation —
|
||||
which is the signal not to match on the title at all.
|
||||
"""
|
||||
if not title:
|
||||
return ""
|
||||
|
||||
folded = _fold(_BRACKETED.sub(" ", title).replace("&", " and "))
|
||||
|
||||
# Repeated because qualifiers stack: "Dune Deluxe Edition Illustrated".
|
||||
while (trimmed := _EDITION_NOISE.sub("", folded)) != folded:
|
||||
folded = trimmed
|
||||
|
||||
# An article says nothing, but a title that is only an article is not improved by
|
||||
# having none, and neither is one that noise removal emptied out.
|
||||
return _LEADING_ARTICLE.sub("", folded, count=1) or folded
|
||||
|
||||
|
||||
def normalize_author(name: str | None) -> str:
|
||||
"""
|
||||
Reduce an author's name to the key their other books should share.
|
||||
|
||||
Deliberately not reduced to surname plus initial: that collides unrelated people,
|
||||
and a wrong match here is a book pointed at a stranger's shelf.
|
||||
|
||||
Args:
|
||||
name: The name as it was stored, in either `Franz Kafka` or `Kafka, Franz` form.
|
||||
|
||||
Returns:
|
||||
The comparison key, or an empty string if nothing survives normalisation.
|
||||
"""
|
||||
if not name:
|
||||
return ""
|
||||
|
||||
# `Kafka, Franz` is one name written backwards. More than one comma is a list, or a
|
||||
# suffix, and guessing at either does more harm than leaving it alone.
|
||||
if name.count(",") == 1:
|
||||
surname, forename = name.split(",")
|
||||
name = f"{forename.strip()} {surname.strip()}"
|
||||
|
||||
folded = _fold(name)
|
||||
|
||||
return _INITIAL_RUN.sub(lambda run: run.group().replace(" ", ""), folded)
|
||||
|
||||
|
||||
def normalize_identifier(name: str, value: str) -> str | None:
|
||||
"""
|
||||
Reduce one identifier to a `scheme:value` key, if it can carry a match at all.
|
||||
|
||||
Args:
|
||||
name: What kind of identifier it is, as stored.
|
||||
value: The identifier itself.
|
||||
|
||||
Returns:
|
||||
The key, or None when the identifier is no use for matching: a per-build UUID,
|
||||
something too short to be evidence, or an ISBN that fails its own checksum.
|
||||
"""
|
||||
name = (name or "").strip().casefold()
|
||||
value = (value or "").strip()
|
||||
|
||||
if not name or not value or name in _PER_BUILD_NAMES:
|
||||
return None
|
||||
|
||||
if name in _ISBN_NAMES:
|
||||
digits = re.sub(r"[^0-9Xx]", "", value).upper()
|
||||
|
||||
if not is_valid_isbn(digits):
|
||||
return None
|
||||
|
||||
# One scheme for both forms: a publisher prints whichever it likes, and the
|
||||
# ISBN-10 and ISBN-13 of an edition are the same number written twice.
|
||||
isbn = digits if len(digits) == 13 else isbn10_to_isbn13(digits)
|
||||
return f"isbn:{isbn}" if isbn else None
|
||||
|
||||
folded = _fold(value) or value.casefold()
|
||||
if len(folded) < _MIN_IDENTIFIER_LENGTH:
|
||||
return None
|
||||
|
||||
return f"{name}:{folded}"
|
||||
@@ -512,6 +512,29 @@ def is_valid_isbn10(isbn: str) -> bool:
|
||||
return str(check_digit) == isbn[-1] or (check_digit == 10 and isbn[-1] in "Xx")
|
||||
|
||||
|
||||
def isbn10_to_isbn13(isbn: str) -> str | None:
|
||||
"""
|
||||
Convert an ISBN-10 to the ISBN-13 naming the same edition.
|
||||
|
||||
The two are the same number written twice: prefix `978`, drop the ISBN-10 check
|
||||
digit, recompute the check digit under the ISBN-13 rule. Matching only works if
|
||||
both forms collapse onto one, since a publisher may print either.
|
||||
|
||||
Args:
|
||||
isbn: A 10-character ISBN, digits and an optional trailing `X` only.
|
||||
|
||||
Returns:
|
||||
The equivalent ISBN-13, or None if the input is not a valid ISBN-10.
|
||||
"""
|
||||
if not is_valid_isbn(isbn) or len(isbn) != 10:
|
||||
return None
|
||||
|
||||
digits = f"978{isbn[:9]}"
|
||||
total = sum(int(digit) * (1 if i % 2 == 0 else 3) for i, digit in enumerate(digits))
|
||||
|
||||
return f"{digits}{(10 - total % 10) % 10}"
|
||||
|
||||
|
||||
def is_valid_isbn13(isbn: str) -> bool:
|
||||
"""
|
||||
Validate an ISBN-13 number using its check digit.
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Tests for the normalization behind book-level duplicate detection."""
|
||||
|
||||
import pytest
|
||||
|
||||
from chitai.services.matching import (
|
||||
normalize_author,
|
||||
normalize_identifier,
|
||||
normalize_title,
|
||||
)
|
||||
from chitai.services.utils import isbn10_to_isbn13
|
||||
|
||||
|
||||
class TestNormalizeTitle:
|
||||
"""Two copies of one book rarely agree on how the title is written."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("title", "expected"),
|
||||
[
|
||||
("The Metamorphosis", "metamorphosis"),
|
||||
("Metamorphosis", "metamorphosis"),
|
||||
("METAMORPHOSIS", "metamorphosis"),
|
||||
("A Tale of Two Cities", "tale of two cities"),
|
||||
("An Enquiry", "enquiry"),
|
||||
# Accents, punctuation and ampersands are spelling, not identity.
|
||||
("Les Misérables", "les miserables"),
|
||||
("Moby Dick; Or, The Whale", "moby dick or the whale"),
|
||||
("Sense & Sensibility", "sense and sensibility"),
|
||||
# Bracketed asides and trailing edition noise say nothing about the book.
|
||||
("Frankenstein (Illustrated)", "frankenstein"),
|
||||
("Frankenstein [Kindle Edition]", "frankenstein"),
|
||||
("Frankenstein, 2nd Edition", "frankenstein"),
|
||||
("Frankenstein Revised Edition", "frankenstein"),
|
||||
("Dune Deluxe Edition Illustrated", "dune"),
|
||||
("", ""),
|
||||
(None, ""),
|
||||
],
|
||||
)
|
||||
def test_titles_that_should_agree(self, title: str | None, expected: str) -> None:
|
||||
assert normalize_title(title) == expected
|
||||
|
||||
def test_a_qualifier_that_is_the_title_survives(self) -> None:
|
||||
"""A trailing qualifier is noise; the same word at the front is the book."""
|
||||
assert normalize_title("The Illustrated Man") == "illustrated man"
|
||||
|
||||
def test_normalization_never_empties_a_title(self) -> None:
|
||||
"""An article-only title is not improved by having no article left."""
|
||||
assert normalize_title("The") == "the"
|
||||
|
||||
|
||||
|
||||
class TestNormalizeAuthor:
|
||||
"""One person, written down several ways."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "expected"),
|
||||
[
|
||||
("Franz Kafka", "franz kafka"),
|
||||
("Kafka, Franz", "franz kafka"),
|
||||
("KAFKA, FRANZ", "franz kafka"),
|
||||
("Émile Zola", "emile zola"),
|
||||
("Doyle, Arthur Conan", "arthur conan doyle"),
|
||||
# Runs of initials are joined, so spacing them out changes nothing.
|
||||
("J.R.R. Tolkien", "jrr tolkien"),
|
||||
("J. R. R. Tolkien", "jrr tolkien"),
|
||||
("JRR Tolkien", "jrr tolkien"),
|
||||
("", ""),
|
||||
(None, ""),
|
||||
],
|
||||
)
|
||||
def test_names_that_should_agree(self, name: str | None, expected: str) -> None:
|
||||
assert normalize_author(name) == expected
|
||||
|
||||
def test_two_people_are_not_reduced_together(self) -> None:
|
||||
"""Surname plus initial would collide unrelated writers; it is not used."""
|
||||
assert normalize_author("Charles Dickens") != normalize_author("Colin Dexter")
|
||||
|
||||
def test_a_list_is_left_alone(self) -> None:
|
||||
"""More than one comma is a list or a suffix, and guessing does more harm."""
|
||||
assert normalize_author("Smith, John, Jr.") == "smith john jr"
|
||||
|
||||
|
||||
class TestNormalizeIdentifier:
|
||||
"""Identifiers only help if the same edition produces the same key."""
|
||||
|
||||
def test_isbn_10_and_isbn_13_are_one_key(self) -> None:
|
||||
assert normalize_identifier("isbn-10", "0486282112") == "isbn:9780486282114"
|
||||
assert normalize_identifier("isbn-13", "9780486282114") == "isbn:9780486282114"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"written", ["978-0-486-28211-4", "978 0 486 28211 4", "9780486282114"]
|
||||
)
|
||||
def test_formatting_is_not_part_of_an_isbn(self, written: str) -> None:
|
||||
assert normalize_identifier("isbn", written) == "isbn:9780486282114"
|
||||
|
||||
def test_an_isbn_that_fails_its_checksum_is_no_evidence(self) -> None:
|
||||
assert normalize_identifier("isbn-13", "9780486282115") is None
|
||||
|
||||
def test_uuids_are_refused(self) -> None:
|
||||
"""Generated per build, so they only re-find what the hash check catches."""
|
||||
assert normalize_identifier("uuid", "3f2b1c4e-1111-2222-3333-444455556666") is None
|
||||
assert normalize_identifier("urn:uuid", "3f2b1c4e-1111-2222-3333-444455556666") is None
|
||||
|
||||
def test_other_schemes_keep_their_own_key(self) -> None:
|
||||
assert normalize_identifier("asin", "B000FC0PDA") == "asin:b000fc0pda"
|
||||
assert normalize_identifier("ASIN", "b000fc0pda") == "asin:b000fc0pda"
|
||||
|
||||
def test_something_too_short_is_not_evidence(self) -> None:
|
||||
"""A Calibre id of "42" would otherwise pair two unrelated books."""
|
||||
assert normalize_identifier("calibre", "42") is None
|
||||
|
||||
@pytest.mark.parametrize(("name", "value"), [("", "1234567"), ("asin", "")])
|
||||
def test_half_an_identifier_is_no_identifier(self, name: str, value: str) -> None:
|
||||
assert normalize_identifier(name, value) is None
|
||||
|
||||
|
||||
class TestIsbnConversion:
|
||||
def test_isbn_10_converts_to_its_isbn_13(self) -> None:
|
||||
assert isbn10_to_isbn13("0486282112") == "9780486282114"
|
||||
|
||||
def test_a_trailing_x_is_a_digit(self) -> None:
|
||||
assert isbn10_to_isbn13("043942089X") == "9780439420891"
|
||||
|
||||
@pytest.mark.parametrize("isbn", ["0486282113", "9780486282114", "nonsense"])
|
||||
def test_anything_that_is_not_an_isbn_10_converts_to_nothing(self, isbn: str) -> None:
|
||||
assert isbn10_to_isbn13(isbn) is None
|
||||
Reference in New Issue
Block a user