fix: match the compact edition markers a cover carries
"Building Microservices, 2E" never matched "Building Microservices". Key the strip on the trailing "e" so 2E, 5e and 3 Ed are caught, while a bare number leaves "Catch 22" and "Blade Runner 2049" alone. Stored keys are recomputed.
This commit is contained in:
@@ -0,0 +1,126 @@
|
||||
"""recompute book matching keys
|
||||
|
||||
Revision ID: ed41acf21270
|
||||
Revises: 4358e7d4743a
|
||||
Create Date: 2026-08-15 15:44:28.341020
|
||||
|
||||
"""
|
||||
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from advanced_alchemy.types import EncryptedString, EncryptedText, GUID, ORA_JSONB, DateTimeUTC, StoredObject, PasswordHash, FernetBackend
|
||||
from advanced_alchemy.types.encrypted_string import PGCryptoBackend
|
||||
from advanced_alchemy.types.password_hash.argon2 import Argon2Hasher
|
||||
from advanced_alchemy.types.password_hash.passlib import PasslibHasher
|
||||
from advanced_alchemy.types.password_hash.pwdlib import PwdlibHasher
|
||||
from sqlalchemy import Text # noqa: F401
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
__all__ = ["downgrade", "upgrade", "schema_upgrades", "schema_downgrades", "data_upgrades", "data_downgrades"]
|
||||
|
||||
sa.GUID = GUID
|
||||
sa.DateTimeUTC = DateTimeUTC
|
||||
sa.ORA_JSONB = ORA_JSONB
|
||||
sa.EncryptedString = EncryptedString
|
||||
sa.EncryptedText = EncryptedText
|
||||
sa.StoredObject = StoredObject
|
||||
sa.PasswordHash = PasswordHash
|
||||
sa.Argon2Hasher = Argon2Hasher
|
||||
sa.PasslibHasher = PasslibHasher
|
||||
sa.PwdlibHasher = PwdlibHasher
|
||||
sa.FernetBackend = FernetBackend
|
||||
sa.PGCryptoBackend = PGCryptoBackend
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'ed41acf21270'
|
||||
down_revision = '4358e7d4743a'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore", category=UserWarning)
|
||||
with op.get_context().autocommit_block():
|
||||
schema_upgrades()
|
||||
data_upgrades()
|
||||
|
||||
def downgrade() -> None:
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore", category=UserWarning)
|
||||
with op.get_context().autocommit_block():
|
||||
data_downgrades()
|
||||
schema_downgrades()
|
||||
|
||||
def schema_upgrades() -> None:
|
||||
"""schema upgrade migrations go here."""
|
||||
pass
|
||||
|
||||
def schema_downgrades() -> None:
|
||||
"""schema downgrade migrations go here."""
|
||||
pass
|
||||
|
||||
def data_upgrades() -> None:
|
||||
"""
|
||||
Recompute every matching key against the current normalization.
|
||||
|
||||
The keys are derived, so changing a helper in `services/matching.py` silently
|
||||
invalidates every row already written — a book stored under the old rules simply
|
||||
stops matching one stored under the new ones, with nothing to show that anything
|
||||
is wrong. `normalize_title` learned to strip the compact edition markers a cover
|
||||
actually carries ("2E", "5e"), which moved `Building Microservices, 2E` onto the
|
||||
same key as `Building Microservices`.
|
||||
|
||||
Any later change to those helpers wants a revision that looks exactly like this
|
||||
one. It is idempotent and safe to re-run.
|
||||
"""
|
||||
from chitai.services.matching import (
|
||||
normalize_author,
|
||||
normalize_identifier,
|
||||
normalize_title,
|
||||
)
|
||||
|
||||
connection = op.get_bind()
|
||||
|
||||
books = connection.execute(sa.text("SELECT id, title FROM books")).fetchall()
|
||||
_apply(
|
||||
connection,
|
||||
"UPDATE books SET normalized_title = :key WHERE id = :id",
|
||||
[{"id": id, "key": normalize_title(title)} for id, title in books],
|
||||
)
|
||||
|
||||
authors = connection.execute(sa.text("SELECT id, name FROM authors")).fetchall()
|
||||
_apply(
|
||||
connection,
|
||||
"UPDATE authors SET normalized_name = :key WHERE id = :id",
|
||||
[{"id": id, "key": normalize_author(name)} for id, name in authors],
|
||||
)
|
||||
|
||||
identifiers = connection.execute(
|
||||
sa.text("SELECT id, name, value FROM identifiers")
|
||||
).fetchall()
|
||||
_apply(
|
||||
connection,
|
||||
"UPDATE identifiers SET normalized_value = :key WHERE id = :id",
|
||||
[
|
||||
{"id": id, "key": normalize_identifier(name, value)}
|
||||
for id, name, value in identifiers
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _apply(connection, statement: str, parameters: list[dict]) -> None:
|
||||
"""Run one update per row, in batches, skipping the work when there are none."""
|
||||
batch_size = 1000
|
||||
|
||||
for start in range(0, len(parameters), batch_size):
|
||||
connection.execute(sa.text(statement), parameters[start : start + batch_size])
|
||||
|
||||
|
||||
def data_downgrades() -> None:
|
||||
"""Add any optional data downgrade migrations here!"""
|
||||
@@ -31,7 +31,10 @@ _BRACKETED = re.compile(r"[(\[{][^)\]}]*[)\]}]")
|
||||
# format note — a qualifier trails the title, it is never the thing the title is about.
|
||||
_EDITION_NOISE = re.compile(
|
||||
r"\s+(?:"
|
||||
r"\d+(?:st|nd|rd|th)?\s+ed(?:ition|n)?"
|
||||
# "2nd edition", but also the compact forms publishers actually print on a
|
||||
# cover: "2E", "3 Ed", "5e". The number alone is never enough — "Catch 22" is
|
||||
# a title and must survive.
|
||||
r"\d+(?:st|nd|rd|th)?\s*(?:edition|edn|ed|e)"
|
||||
r"|(?:first|second|third|fourth|fifth|sixth|new|revised|expanded|updated|"
|
||||
r"annotated|illustrated|unabridged|abridged|complete|definitive|deluxe|"
|
||||
r"anniversary|collectors|international|kindle|paperback|hardcover|hardback|"
|
||||
|
||||
@@ -30,6 +30,11 @@ class TestNormalizeTitle:
|
||||
("Frankenstein (Illustrated)", "frankenstein"),
|
||||
("Frankenstein [Kindle Edition]", "frankenstein"),
|
||||
("Frankenstein, 2nd Edition", "frankenstein"),
|
||||
# The compact forms a cover actually carries.
|
||||
("Building Microservices, 2E", "building microservices"),
|
||||
("Building Microservices 2e", "building microservices"),
|
||||
("Frankenstein 3 Ed", "frankenstein"),
|
||||
("Dungeons & Dragons 5e", "dungeons and dragons"),
|
||||
("Frankenstein Revised Edition", "frankenstein"),
|
||||
("Dune Deluxe Edition Illustrated", "dune"),
|
||||
("", ""),
|
||||
@@ -47,6 +52,13 @@ class TestNormalizeTitle:
|
||||
"""An article-only title is not improved by having no article left."""
|
||||
assert normalize_title("The") == "the"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"title", ["Catch 22", "Fahrenheit 451", "Blade Runner 2049", "1984", "Apollo 13"]
|
||||
)
|
||||
def test_a_number_is_not_an_edition(self, title: str) -> None:
|
||||
"""Edition stripping keys on the `e`; a bare number is part of the title."""
|
||||
assert normalize_title(title) == title.casefold()
|
||||
|
||||
|
||||
|
||||
class TestNormalizeAuthor:
|
||||
|
||||
Reference in New Issue
Block a user