From 373c96d6d66fddd91de040ad1cf1105491f8cb83 Mon Sep 17 00:00:00 2001 From: patrick Date: Sat, 15 Aug 2026 21:55:45 -0400 Subject: [PATCH] fix: match the compact edition markers a cover carries "Building Microservices, 2E" never matched "Building Microservices". Key the strip on the trailing "e" so 2E, 5e and 3 Ed are caught, while a bare number leaves "Catch 22" and "Blade Runner 2049" alone. Stored keys are recomputed. --- ...compute_book_matching_keys_ed41acf21270.py | 126 ++++++++++++++++++ backend/src/chitai/services/matching.py | 5 +- backend/tests/unit/test_matching.py | 12 ++ 3 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 backend/migrations/versions/2026-08-15_recompute_book_matching_keys_ed41acf21270.py diff --git a/backend/migrations/versions/2026-08-15_recompute_book_matching_keys_ed41acf21270.py b/backend/migrations/versions/2026-08-15_recompute_book_matching_keys_ed41acf21270.py new file mode 100644 index 0000000..a32f8e8 --- /dev/null +++ b/backend/migrations/versions/2026-08-15_recompute_book_matching_keys_ed41acf21270.py @@ -0,0 +1,126 @@ +"""recompute book matching keys + +Revision ID: ed41acf21270 +Revises: 4358e7d4743a +Create Date: 2026-08-15 15:44:28.341020 + +""" + +import warnings +from typing import TYPE_CHECKING + +import sqlalchemy as sa +from alembic import op +from advanced_alchemy.types import EncryptedString, EncryptedText, GUID, ORA_JSONB, DateTimeUTC, StoredObject, PasswordHash, FernetBackend +from advanced_alchemy.types.encrypted_string import PGCryptoBackend +from advanced_alchemy.types.password_hash.argon2 import Argon2Hasher +from advanced_alchemy.types.password_hash.passlib import PasslibHasher +from advanced_alchemy.types.password_hash.pwdlib import PwdlibHasher +from sqlalchemy import Text # noqa: F401 + +if TYPE_CHECKING: + from collections.abc import Sequence + +__all__ = ["downgrade", "upgrade", "schema_upgrades", "schema_downgrades", "data_upgrades", "data_downgrades"] + +sa.GUID = GUID +sa.DateTimeUTC = DateTimeUTC +sa.ORA_JSONB = ORA_JSONB +sa.EncryptedString = EncryptedString +sa.EncryptedText = EncryptedText +sa.StoredObject = StoredObject +sa.PasswordHash = PasswordHash +sa.Argon2Hasher = Argon2Hasher +sa.PasslibHasher = PasslibHasher +sa.PwdlibHasher = PwdlibHasher +sa.FernetBackend = FernetBackend +sa.PGCryptoBackend = PGCryptoBackend + +# revision identifiers, used by Alembic. +revision = 'ed41acf21270' +down_revision = '4358e7d4743a' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + with op.get_context().autocommit_block(): + schema_upgrades() + data_upgrades() + +def downgrade() -> None: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + with op.get_context().autocommit_block(): + data_downgrades() + schema_downgrades() + +def schema_upgrades() -> None: + """schema upgrade migrations go here.""" + pass + +def schema_downgrades() -> None: + """schema downgrade migrations go here.""" + pass + +def data_upgrades() -> None: + """ + Recompute every matching key against the current normalization. + + The keys are derived, so changing a helper in `services/matching.py` silently + invalidates every row already written — a book stored under the old rules simply + stops matching one stored under the new ones, with nothing to show that anything + is wrong. `normalize_title` learned to strip the compact edition markers a cover + actually carries ("2E", "5e"), which moved `Building Microservices, 2E` onto the + same key as `Building Microservices`. + + Any later change to those helpers wants a revision that looks exactly like this + one. It is idempotent and safe to re-run. + """ + from chitai.services.matching import ( + normalize_author, + normalize_identifier, + normalize_title, + ) + + connection = op.get_bind() + + books = connection.execute(sa.text("SELECT id, title FROM books")).fetchall() + _apply( + connection, + "UPDATE books SET normalized_title = :key WHERE id = :id", + [{"id": id, "key": normalize_title(title)} for id, title in books], + ) + + authors = connection.execute(sa.text("SELECT id, name FROM authors")).fetchall() + _apply( + connection, + "UPDATE authors SET normalized_name = :key WHERE id = :id", + [{"id": id, "key": normalize_author(name)} for id, name in authors], + ) + + identifiers = connection.execute( + sa.text("SELECT id, name, value FROM identifiers") + ).fetchall() + _apply( + connection, + "UPDATE identifiers SET normalized_value = :key WHERE id = :id", + [ + {"id": id, "key": normalize_identifier(name, value)} + for id, name, value in identifiers + ], + ) + + +def _apply(connection, statement: str, parameters: list[dict]) -> None: + """Run one update per row, in batches, skipping the work when there are none.""" + batch_size = 1000 + + for start in range(0, len(parameters), batch_size): + connection.execute(sa.text(statement), parameters[start : start + batch_size]) + + +def data_downgrades() -> None: + """Add any optional data downgrade migrations here!""" diff --git a/backend/src/chitai/services/matching.py b/backend/src/chitai/services/matching.py index 3dd9bd5..b0222b0 100644 --- a/backend/src/chitai/services/matching.py +++ b/backend/src/chitai/services/matching.py @@ -31,7 +31,10 @@ _BRACKETED = re.compile(r"[(\[{][^)\]}]*[)\]}]") # format note — a qualifier trails the title, it is never the thing the title is about. _EDITION_NOISE = re.compile( r"\s+(?:" - r"\d+(?:st|nd|rd|th)?\s+ed(?:ition|n)?" + # "2nd edition", but also the compact forms publishers actually print on a + # cover: "2E", "3 Ed", "5e". The number alone is never enough — "Catch 22" is + # a title and must survive. + r"\d+(?:st|nd|rd|th)?\s*(?:edition|edn|ed|e)" r"|(?:first|second|third|fourth|fifth|sixth|new|revised|expanded|updated|" r"annotated|illustrated|unabridged|abridged|complete|definitive|deluxe|" r"anniversary|collectors|international|kindle|paperback|hardcover|hardback|" diff --git a/backend/tests/unit/test_matching.py b/backend/tests/unit/test_matching.py index 55abe13..7230359 100644 --- a/backend/tests/unit/test_matching.py +++ b/backend/tests/unit/test_matching.py @@ -30,6 +30,11 @@ class TestNormalizeTitle: ("Frankenstein (Illustrated)", "frankenstein"), ("Frankenstein [Kindle Edition]", "frankenstein"), ("Frankenstein, 2nd Edition", "frankenstein"), + # The compact forms a cover actually carries. + ("Building Microservices, 2E", "building microservices"), + ("Building Microservices 2e", "building microservices"), + ("Frankenstein 3 Ed", "frankenstein"), + ("Dungeons & Dragons 5e", "dungeons and dragons"), ("Frankenstein Revised Edition", "frankenstein"), ("Dune Deluxe Edition Illustrated", "dune"), ("", ""), @@ -47,6 +52,13 @@ class TestNormalizeTitle: """An article-only title is not improved by having no article left.""" assert normalize_title("The") == "the" + @pytest.mark.parametrize( + "title", ["Catch 22", "Fahrenheit 451", "Blade Runner 2049", "1984", "Apollo 13"] + ) + def test_a_number_is_not_an_edition(self, title: str) -> None: + """Edition stripping keys on the `e`; a bare number is part of the title.""" + assert normalize_title(title) == title.casefold() + class TestNormalizeAuthor: