feat: store matching keys and duplicate dismissals

Derive normalized_title, normalized_name and normalized_value with @validates
so no write can bypass them, and add the table recording pairs a reader has
said are not the same book.

backend/migrations/versions/2026-08-15_add_book_matching_keys_and_duplicate__4358e7d4743a.py

@@ -0,0 +1,167 @@
+"""add book matching keys and duplicate dismissals
+
+Revision ID: 4358e7d4743a
+Revises: e9c2c7e875ae
+Create Date: 2026-08-15 15:09:02.708914
+
+"""
+
+import warnings
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+from advanced_alchemy.types import EncryptedString, EncryptedText, GUID, ORA_JSONB, DateTimeUTC, StoredObject, PasswordHash, FernetBackend
+from advanced_alchemy.types.encrypted_string import PGCryptoBackend
+from advanced_alchemy.types.password_hash.argon2 import Argon2Hasher
+from advanced_alchemy.types.password_hash.passlib import PasslibHasher
+from advanced_alchemy.types.password_hash.pwdlib import PwdlibHasher
+from sqlalchemy import Text  # noqa: F401
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+__all__ = ["downgrade", "upgrade", "schema_upgrades", "schema_downgrades", "data_upgrades", "data_downgrades"]
+
+sa.GUID = GUID
+sa.DateTimeUTC = DateTimeUTC
+sa.ORA_JSONB = ORA_JSONB
+sa.EncryptedString = EncryptedString
+sa.EncryptedText = EncryptedText
+sa.StoredObject = StoredObject
+sa.PasswordHash = PasswordHash
+sa.Argon2Hasher = Argon2Hasher
+sa.PasslibHasher = PasslibHasher
+sa.PwdlibHasher = PwdlibHasher
+sa.FernetBackend = FernetBackend
+sa.PGCryptoBackend = PGCryptoBackend
+
+# revision identifiers, used by Alembic.
+revision = '4358e7d4743a'
+down_revision = 'e9c2c7e875ae'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", category=UserWarning)
+        with op.get_context().autocommit_block():
+            schema_upgrades()
+            data_upgrades()
+
+def downgrade() -> None:
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", category=UserWarning)
+        with op.get_context().autocommit_block():
+            data_downgrades()
+            schema_downgrades()
+
+def schema_upgrades() -> None:
+    """schema upgrade migrations go here."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('duplicate_dismissals',
+    sa.Column('id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), nullable=False),
+    sa.Column('book_a_id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), nullable=False),
+    sa.Column('book_b_id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), nullable=False),
+    sa.ForeignKeyConstraint(['book_a_id'], ['books.id'], name=op.f('fk_duplicate_dismissals_book_a_id_books'), ondelete='cascade'),
+    sa.ForeignKeyConstraint(['book_b_id'], ['books.id'], name=op.f('fk_duplicate_dismissals_book_b_id_books'), ondelete='cascade'),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_duplicate_dismissals')),
+    sa.UniqueConstraint('book_a_id', 'book_b_id', name=op.f('uq_duplicate_dismissals_book_a_id'))
+    )
+    with op.batch_alter_table('duplicate_dismissals', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_duplicate_dismissals_book_a_id'), ['book_a_id'], unique=False)
+        batch_op.create_index(batch_op.f('ix_duplicate_dismissals_book_b_id'), ['book_b_id'], unique=False)
+
+    # `server_default` so the column can be added to a table that already has rows;
+    # `data_upgrades` fills in the real keys immediately afterwards.
+    with op.batch_alter_table('authors', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('normalized_name', sa.String(), nullable=False, server_default=''))
+        batch_op.create_index(batch_op.f('ix_authors_normalized_name'), ['normalized_name'], unique=False)
+
+    with op.batch_alter_table('books', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('normalized_title', sa.String(), nullable=False, server_default=''))
+        batch_op.create_index(batch_op.f('ix_books_normalized_title'), ['normalized_title'], unique=False)
+
+    with op.batch_alter_table('identifiers', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('normalized_value', sa.String(), nullable=True))
+        batch_op.create_index(batch_op.f('ix_identifiers_normalized_value'), ['normalized_value'], unique=False)
+
+    # ### end Alembic commands ###
+
+def schema_downgrades() -> None:
+    """schema downgrade migrations go here."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('identifiers', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_identifiers_normalized_value'))
+        batch_op.drop_column('normalized_value')
+
+    with op.batch_alter_table('books', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_books_normalized_title'))
+        batch_op.drop_column('normalized_title')
+
+    with op.batch_alter_table('authors', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_authors_normalized_name'))
+        batch_op.drop_column('normalized_name')
+
+    with op.batch_alter_table('duplicate_dismissals', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_duplicate_dismissals_book_b_id'))
+        batch_op.drop_index(batch_op.f('ix_duplicate_dismissals_book_a_id'))
+
+    op.drop_table('duplicate_dismissals')
+    # ### end Alembic commands ###
+
+def data_upgrades() -> None:
+    """
+    Fill the matching keys in for rows that already exist.
+
+    The validators on the models only fire when something is written, so without this
+    every book imported before today is invisible to duplicate detection. Run through
+    the same helpers the validators use, so a backfilled row and a freshly written one
+    are guaranteed to agree.
+    """
+    from chitai.services.matching import (
+        normalize_author,
+        normalize_identifier,
+        normalize_title,
+    )
+
+    connection = op.get_bind()
+
+    books = connection.execute(sa.text("SELECT id, title FROM books")).fetchall()
+    _apply(
+        connection,
+        "UPDATE books SET normalized_title = :key WHERE id = :id",
+        [{"id": id, "key": normalize_title(title)} for id, title in books],
+    )
+
+    authors = connection.execute(sa.text("SELECT id, name FROM authors")).fetchall()
+    _apply(
+        connection,
+        "UPDATE authors SET normalized_name = :key WHERE id = :id",
+        [{"id": id, "key": normalize_author(name)} for id, name in authors],
+    )
+
+    identifiers = connection.execute(
+        sa.text("SELECT id, name, value FROM identifiers")
+    ).fetchall()
+    _apply(
+        connection,
+        "UPDATE identifiers SET normalized_value = :key WHERE id = :id",
+        [
+            {"id": id, "key": normalize_identifier(name, value)}
+            for id, name, value in identifiers
+        ],
+    )
+
+
+def _apply(connection, statement: str, parameters: list[dict]) -> None:
+    """Run one update per row, in batches, skipping the work when there are none."""
+    batch_size = 1000
+
+    for start in range(0, len(parameters), batch_size):
+        connection.execute(sa.text(statement), parameters[start : start + batch_size])
+
+
+def data_downgrades() -> None:
+    """Add any optional data downgrade migrations here!"""

backend/src/chitai/database/models/__init__.py

@@ -3,6 +3,7 @@ from .book import Book, Identifier, FileMetadata
 from .book_list import BookList, BookListLink
 from .book_progress import BookProgress
 from .book_series import BookSeries
+from .duplicate_dismissal import DuplicateDismissal
 from .kosync_device import KosyncDevice
 from .kosync_progress import KosyncProgress
 from .library import Library

backend/src/chitai/database/models/author.py

@@ -5,6 +5,7 @@ from sqlalchemy import ColumnElement, ForeignKey, UniqueConstraint
 from sqlalchemy.orm import Mapped
 from sqlalchemy.orm import mapped_column
 from sqlalchemy.orm import relationship
+from sqlalchemy.orm import validates
 
 from advanced_alchemy.base import BigIntAuditBase, BigIntBase
 from advanced_alchemy.mixins import UniqueMixin
@@ -17,8 +18,25 @@ class Author(BigIntAuditBase, UniqueMixin):
     __tablename__ = "authors"
 
     name: Mapped[str] = mapped_column(unique=True, index=True)
+
+    # Kept current by `_normalize_name` below — never assign it directly. Not unique:
+    # `Kafka, Franz` and `Franz Kafka` are two rows here and one person to a reader,
+    # which is exactly what this column exists to express.
+    normalized_name: Mapped[str] = mapped_column(default="", index=True)
+
     description: Mapped[Optional[str]]
 
+    @validates("name")
+    def _normalize_name(self, _key: str, name: str) -> str:
+        """Derive `normalized_name` from whatever writes the name."""
+        # Imported here rather than at module scope: `chitai.services.matching` cannot
+        # be reached without initialising the `chitai.services` package, which imports
+        # the services, which import this module.
+        from chitai.services.matching import normalize_author
+
+        self.normalized_name = normalize_author(name)
+        return name
+
     @classmethod
     def unique_hash(cls, name: str) -> Hashable:
         """Generate a unique hash for deduplication."""

backend/src/chitai/database/models/book.py

@@ -2,7 +2,7 @@ from datetime import date
 from typing import TYPE_CHECKING, Any, Optional
 
 from sqlalchemy import Index, ForeignKey, UniqueConstraint
-from sqlalchemy.orm import Mapped, mapped_column, relationship
+from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
 from sqlalchemy.orm import mapped_column
 from sqlalchemy.orm import relationship
 from sqlalchemy.ext.orderinglist import ordering_list
@@ -44,6 +44,13 @@ class Book(BigIntAuditBase):
     library: Mapped["Library"] = relationship(back_populates="books")
 
     title: Mapped[str]
+
+    # Kept current by `_normalize_title` below — never assign it directly.
+    #
+    # Deliberately not unique: two spellings collapsing onto one value is the whole
+    # point of the column, and a second edition is allowed to exist.
+    normalized_title: Mapped[str] = mapped_column(default="", index=True)
+
     subtitle: Mapped[Optional[str]]
     description: Mapped[Optional[str]]
     published_date: Mapped[Optional[date]]
@@ -111,6 +118,24 @@ class Book(BigIntAuditBase):
     def progress(self) -> Optional["BookProgress"]:
         return self.progress_records[0] if self.progress_records else None
 
+    @validates("title")
+    def _normalize_title(self, _key: str, title: str) -> str:
+        """
+        Derive `normalized_title` from whatever writes the title.
+
+        A validator rather than a service call because `BookService` sets titles from
+        at least three places — `to_model_on_create`, `to_model_on_update` and the
+        `setattr` loop in `_populate_with_unique_relationships` — and a fourth would
+        otherwise leave the key silently stale.
+        """
+        # Imported here rather than at module scope: `chitai.services.matching` cannot
+        # be reached without initialising the `chitai.services` package, which imports
+        # the services, which import this module.
+        from chitai.services.matching import normalize_title
+
+        self.normalized_title = normalize_title(title)
+        return title
+
     def __repr__(self) -> str:
         return f"Book({self.title=!r})"
 
@@ -132,6 +157,24 @@ class Identifier(BigIntBase):
     book_id: Mapped[int] = mapped_column(ForeignKey("books.id", ondelete="cascade"))
     value: Mapped[str]
 
+    # Kept current by `_normalize` below — never assign it directly. Null for an
+    # identifier that cannot carry a match: a per-build UUID, or an ISBN that fails
+    # its own checksum.
+    normalized_value: Mapped[Optional[str]] = mapped_column(index=True)
+
+    @validates("name", "value")
+    def _normalize(self, key: str, value: str) -> str:
+        """Recompute `normalized_value` whenever either half of the pair changes."""
+        from chitai.services.matching import normalize_identifier
+
+        name = value if key == "name" else self.name
+        raw = value if key == "value" else self.value
+
+        self.normalized_value = (
+            normalize_identifier(name, raw) if name and raw else None
+        )
+        return value
+
     def __repr__(self):
         return f"Identifier({self.name!r} : {self.value!r})"
 

backend/src/chitai/database/models/duplicate_dismissal.py

@@ -0,0 +1,37 @@
+from sqlalchemy import ForeignKey, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column
+
+from advanced_alchemy.base import BigIntBase
+
+
+class DuplicateDismissal(BigIntBase):
+    """
+    Two books a reader has said are not the same book.
+
+    Title and author matching is probabilistic, so it will keep proposing a second
+    edition, a translation and a sequel that shares its predecessor's name. A review
+    screen with no way to disagree with it nags forever, which is how people learn to
+    ignore a screen.
+
+    The pair is stored ordered — `book_a_id` is always the lower id — so "A and B" and
+    "B and A" are one row and the unique constraint can do its job. Use `pair()` rather
+    than assigning the columns directly.
+    """
+
+    __tablename__ = "duplicate_dismissals"
+    __table_args__ = (UniqueConstraint("book_a_id", "book_b_id"),)
+
+    book_a_id: Mapped[int] = mapped_column(
+        ForeignKey("books.id", ondelete="cascade"), index=True
+    )
+    book_b_id: Mapped[int] = mapped_column(
+        ForeignKey("books.id", ondelete="cascade"), index=True
+    )
+
+    @staticmethod
+    def pair(first: int, second: int) -> tuple[int, int]:
+        """The two book ids in the order this table stores them."""
+        return (first, second) if first <= second else (second, first)
+
+    def __repr__(self) -> str:
+        return f"DuplicateDismissal({self.book_a_id!r}, {self.book_b_id!r})"