diff --git a/backend/migrations/versions/2026-08-15_add_book_matching_keys_and_duplicate__4358e7d4743a.py b/backend/migrations/versions/2026-08-15_add_book_matching_keys_and_duplicate__4358e7d4743a.py new file mode 100644 index 0000000..4d87667 --- /dev/null +++ b/backend/migrations/versions/2026-08-15_add_book_matching_keys_and_duplicate__4358e7d4743a.py @@ -0,0 +1,167 @@ +"""add book matching keys and duplicate dismissals + +Revision ID: 4358e7d4743a +Revises: e9c2c7e875ae +Create Date: 2026-08-15 15:09:02.708914 + +""" + +import warnings +from typing import TYPE_CHECKING + +import sqlalchemy as sa +from alembic import op +from advanced_alchemy.types import EncryptedString, EncryptedText, GUID, ORA_JSONB, DateTimeUTC, StoredObject, PasswordHash, FernetBackend +from advanced_alchemy.types.encrypted_string import PGCryptoBackend +from advanced_alchemy.types.password_hash.argon2 import Argon2Hasher +from advanced_alchemy.types.password_hash.passlib import PasslibHasher +from advanced_alchemy.types.password_hash.pwdlib import PwdlibHasher +from sqlalchemy import Text # noqa: F401 + +if TYPE_CHECKING: + from collections.abc import Sequence + +__all__ = ["downgrade", "upgrade", "schema_upgrades", "schema_downgrades", "data_upgrades", "data_downgrades"] + +sa.GUID = GUID +sa.DateTimeUTC = DateTimeUTC +sa.ORA_JSONB = ORA_JSONB +sa.EncryptedString = EncryptedString +sa.EncryptedText = EncryptedText +sa.StoredObject = StoredObject +sa.PasswordHash = PasswordHash +sa.Argon2Hasher = Argon2Hasher +sa.PasslibHasher = PasslibHasher +sa.PwdlibHasher = PwdlibHasher +sa.FernetBackend = FernetBackend +sa.PGCryptoBackend = PGCryptoBackend + +# revision identifiers, used by Alembic. +revision = '4358e7d4743a' +down_revision = 'e9c2c7e875ae' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + with op.get_context().autocommit_block(): + schema_upgrades() + data_upgrades() + +def downgrade() -> None: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + with op.get_context().autocommit_block(): + data_downgrades() + schema_downgrades() + +def schema_upgrades() -> None: + """schema upgrade migrations go here.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('duplicate_dismissals', + sa.Column('id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), nullable=False), + sa.Column('book_a_id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), nullable=False), + sa.Column('book_b_id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), nullable=False), + sa.ForeignKeyConstraint(['book_a_id'], ['books.id'], name=op.f('fk_duplicate_dismissals_book_a_id_books'), ondelete='cascade'), + sa.ForeignKeyConstraint(['book_b_id'], ['books.id'], name=op.f('fk_duplicate_dismissals_book_b_id_books'), ondelete='cascade'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_duplicate_dismissals')), + sa.UniqueConstraint('book_a_id', 'book_b_id', name=op.f('uq_duplicate_dismissals_book_a_id')) + ) + with op.batch_alter_table('duplicate_dismissals', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_duplicate_dismissals_book_a_id'), ['book_a_id'], unique=False) + batch_op.create_index(batch_op.f('ix_duplicate_dismissals_book_b_id'), ['book_b_id'], unique=False) + + # `server_default` so the column can be added to a table that already has rows; + # `data_upgrades` fills in the real keys immediately afterwards. + with op.batch_alter_table('authors', schema=None) as batch_op: + batch_op.add_column(sa.Column('normalized_name', sa.String(), nullable=False, server_default='')) + batch_op.create_index(batch_op.f('ix_authors_normalized_name'), ['normalized_name'], unique=False) + + with op.batch_alter_table('books', schema=None) as batch_op: + batch_op.add_column(sa.Column('normalized_title', sa.String(), nullable=False, server_default='')) + batch_op.create_index(batch_op.f('ix_books_normalized_title'), ['normalized_title'], unique=False) + + with op.batch_alter_table('identifiers', schema=None) as batch_op: + batch_op.add_column(sa.Column('normalized_value', sa.String(), nullable=True)) + batch_op.create_index(batch_op.f('ix_identifiers_normalized_value'), ['normalized_value'], unique=False) + + # ### end Alembic commands ### + +def schema_downgrades() -> None: + """schema downgrade migrations go here.""" + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('identifiers', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_identifiers_normalized_value')) + batch_op.drop_column('normalized_value') + + with op.batch_alter_table('books', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_books_normalized_title')) + batch_op.drop_column('normalized_title') + + with op.batch_alter_table('authors', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_authors_normalized_name')) + batch_op.drop_column('normalized_name') + + with op.batch_alter_table('duplicate_dismissals', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_duplicate_dismissals_book_b_id')) + batch_op.drop_index(batch_op.f('ix_duplicate_dismissals_book_a_id')) + + op.drop_table('duplicate_dismissals') + # ### end Alembic commands ### + +def data_upgrades() -> None: + """ + Fill the matching keys in for rows that already exist. + + The validators on the models only fire when something is written, so without this + every book imported before today is invisible to duplicate detection. Run through + the same helpers the validators use, so a backfilled row and a freshly written one + are guaranteed to agree. + """ + from chitai.services.matching import ( + normalize_author, + normalize_identifier, + normalize_title, + ) + + connection = op.get_bind() + + books = connection.execute(sa.text("SELECT id, title FROM books")).fetchall() + _apply( + connection, + "UPDATE books SET normalized_title = :key WHERE id = :id", + [{"id": id, "key": normalize_title(title)} for id, title in books], + ) + + authors = connection.execute(sa.text("SELECT id, name FROM authors")).fetchall() + _apply( + connection, + "UPDATE authors SET normalized_name = :key WHERE id = :id", + [{"id": id, "key": normalize_author(name)} for id, name in authors], + ) + + identifiers = connection.execute( + sa.text("SELECT id, name, value FROM identifiers") + ).fetchall() + _apply( + connection, + "UPDATE identifiers SET normalized_value = :key WHERE id = :id", + [ + {"id": id, "key": normalize_identifier(name, value)} + for id, name, value in identifiers + ], + ) + + +def _apply(connection, statement: str, parameters: list[dict]) -> None: + """Run one update per row, in batches, skipping the work when there are none.""" + batch_size = 1000 + + for start in range(0, len(parameters), batch_size): + connection.execute(sa.text(statement), parameters[start : start + batch_size]) + + +def data_downgrades() -> None: + """Add any optional data downgrade migrations here!""" diff --git a/backend/src/chitai/database/models/__init__.py b/backend/src/chitai/database/models/__init__.py index 16df64f..01a568a 100644 --- a/backend/src/chitai/database/models/__init__.py +++ b/backend/src/chitai/database/models/__init__.py @@ -3,6 +3,7 @@ from .book import Book, Identifier, FileMetadata from .book_list import BookList, BookListLink from .book_progress import BookProgress from .book_series import BookSeries +from .duplicate_dismissal import DuplicateDismissal from .kosync_device import KosyncDevice from .kosync_progress import KosyncProgress from .library import Library diff --git a/backend/src/chitai/database/models/author.py b/backend/src/chitai/database/models/author.py index b47fe4a..1e2f111 100644 --- a/backend/src/chitai/database/models/author.py +++ b/backend/src/chitai/database/models/author.py @@ -5,6 +5,7 @@ from sqlalchemy import ColumnElement, ForeignKey, UniqueConstraint from sqlalchemy.orm import Mapped from sqlalchemy.orm import mapped_column from sqlalchemy.orm import relationship +from sqlalchemy.orm import validates from advanced_alchemy.base import BigIntAuditBase, BigIntBase from advanced_alchemy.mixins import UniqueMixin @@ -17,8 +18,25 @@ class Author(BigIntAuditBase, UniqueMixin): __tablename__ = "authors" name: Mapped[str] = mapped_column(unique=True, index=True) + + # Kept current by `_normalize_name` below — never assign it directly. Not unique: + # `Kafka, Franz` and `Franz Kafka` are two rows here and one person to a reader, + # which is exactly what this column exists to express. + normalized_name: Mapped[str] = mapped_column(default="", index=True) + description: Mapped[Optional[str]] + @validates("name") + def _normalize_name(self, _key: str, name: str) -> str: + """Derive `normalized_name` from whatever writes the name.""" + # Imported here rather than at module scope: `chitai.services.matching` cannot + # be reached without initialising the `chitai.services` package, which imports + # the services, which import this module. + from chitai.services.matching import normalize_author + + self.normalized_name = normalize_author(name) + return name + @classmethod def unique_hash(cls, name: str) -> Hashable: """Generate a unique hash for deduplication.""" diff --git a/backend/src/chitai/database/models/book.py b/backend/src/chitai/database/models/book.py index 6b75167..80665b6 100644 --- a/backend/src/chitai/database/models/book.py +++ b/backend/src/chitai/database/models/book.py @@ -2,7 +2,7 @@ from datetime import date from typing import TYPE_CHECKING, Any, Optional from sqlalchemy import Index, ForeignKey, UniqueConstraint -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, mapped_column, relationship, validates from sqlalchemy.orm import mapped_column from sqlalchemy.orm import relationship from sqlalchemy.ext.orderinglist import ordering_list @@ -44,6 +44,13 @@ class Book(BigIntAuditBase): library: Mapped["Library"] = relationship(back_populates="books") title: Mapped[str] + + # Kept current by `_normalize_title` below — never assign it directly. + # + # Deliberately not unique: two spellings collapsing onto one value is the whole + # point of the column, and a second edition is allowed to exist. + normalized_title: Mapped[str] = mapped_column(default="", index=True) + subtitle: Mapped[Optional[str]] description: Mapped[Optional[str]] published_date: Mapped[Optional[date]] @@ -111,6 +118,24 @@ class Book(BigIntAuditBase): def progress(self) -> Optional["BookProgress"]: return self.progress_records[0] if self.progress_records else None + @validates("title") + def _normalize_title(self, _key: str, title: str) -> str: + """ + Derive `normalized_title` from whatever writes the title. + + A validator rather than a service call because `BookService` sets titles from + at least three places — `to_model_on_create`, `to_model_on_update` and the + `setattr` loop in `_populate_with_unique_relationships` — and a fourth would + otherwise leave the key silently stale. + """ + # Imported here rather than at module scope: `chitai.services.matching` cannot + # be reached without initialising the `chitai.services` package, which imports + # the services, which import this module. + from chitai.services.matching import normalize_title + + self.normalized_title = normalize_title(title) + return title + def __repr__(self) -> str: return f"Book({self.title=!r})" @@ -132,6 +157,24 @@ class Identifier(BigIntBase): book_id: Mapped[int] = mapped_column(ForeignKey("books.id", ondelete="cascade")) value: Mapped[str] + # Kept current by `_normalize` below — never assign it directly. Null for an + # identifier that cannot carry a match: a per-build UUID, or an ISBN that fails + # its own checksum. + normalized_value: Mapped[Optional[str]] = mapped_column(index=True) + + @validates("name", "value") + def _normalize(self, key: str, value: str) -> str: + """Recompute `normalized_value` whenever either half of the pair changes.""" + from chitai.services.matching import normalize_identifier + + name = value if key == "name" else self.name + raw = value if key == "value" else self.value + + self.normalized_value = ( + normalize_identifier(name, raw) if name and raw else None + ) + return value + def __repr__(self): return f"Identifier({self.name!r} : {self.value!r})" diff --git a/backend/src/chitai/database/models/duplicate_dismissal.py b/backend/src/chitai/database/models/duplicate_dismissal.py new file mode 100644 index 0000000..0aa1eeb --- /dev/null +++ b/backend/src/chitai/database/models/duplicate_dismissal.py @@ -0,0 +1,37 @@ +from sqlalchemy import ForeignKey, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from advanced_alchemy.base import BigIntBase + + +class DuplicateDismissal(BigIntBase): + """ + Two books a reader has said are not the same book. + + Title and author matching is probabilistic, so it will keep proposing a second + edition, a translation and a sequel that shares its predecessor's name. A review + screen with no way to disagree with it nags forever, which is how people learn to + ignore a screen. + + The pair is stored ordered — `book_a_id` is always the lower id — so "A and B" and + "B and A" are one row and the unique constraint can do its job. Use `pair()` rather + than assigning the columns directly. + """ + + __tablename__ = "duplicate_dismissals" + __table_args__ = (UniqueConstraint("book_a_id", "book_b_id"),) + + book_a_id: Mapped[int] = mapped_column( + ForeignKey("books.id", ondelete="cascade"), index=True + ) + book_b_id: Mapped[int] = mapped_column( + ForeignKey("books.id", ondelete="cascade"), index=True + ) + + @staticmethod + def pair(first: int, second: int) -> tuple[int, int]: + """The two book ids in the order this table stores them.""" + return (first, second) if first <= second else (second, first) + + def __repr__(self) -> str: + return f"DuplicateDismissal({self.book_a_id!r}, {self.book_b_id!r})"