patrick
8ee406533c
Extractors wrote whatever the file said, so one person held several rows: "Sam Newman" beside "Newman, Sam;" beside "Sam Newman.epub", the last because the upload path never stripped the file extension. Tidy on write, in the validator and in the uniqueness lookup alike, and merge the rows that collide.
235 lines
9.8 KiB
Python
235 lines
9.8 KiB
Python
"""Tests for the normalization behind book-level duplicate detection."""
|
|
|
|
import pytest
|
|
|
|
from chitai.services.matching import (
|
|
format_author_name,
|
|
normalize_author,
|
|
normalize_identifier,
|
|
normalize_title,
|
|
)
|
|
from chitai.services.metadata_extractor import parse_identifier
|
|
from chitai.services.utils import isbn10_to_isbn13
|
|
|
|
|
|
class TestNormalizeTitle:
|
|
"""Two copies of one book rarely agree on how the title is written."""
|
|
|
|
@pytest.mark.parametrize(
|
|
("title", "expected"),
|
|
[
|
|
("The Metamorphosis", "metamorphosis"),
|
|
("Metamorphosis", "metamorphosis"),
|
|
("METAMORPHOSIS", "metamorphosis"),
|
|
("A Tale of Two Cities", "tale of two cities"),
|
|
("An Enquiry", "enquiry"),
|
|
# Accents, punctuation and ampersands are spelling, not identity.
|
|
("Les Misérables", "les miserables"),
|
|
("Moby Dick; Or, The Whale", "moby dick or the whale"),
|
|
("Sense & Sensibility", "sense and sensibility"),
|
|
# Bracketed asides and trailing edition noise say nothing about the book.
|
|
("Frankenstein (Illustrated)", "frankenstein"),
|
|
("Frankenstein [Kindle Edition]", "frankenstein"),
|
|
("Frankenstein, 2nd Edition", "frankenstein"),
|
|
# The compact forms a cover actually carries.
|
|
("Building Microservices, 2E", "building microservices"),
|
|
("Building Microservices 2e", "building microservices"),
|
|
("Frankenstein 3 Ed", "frankenstein"),
|
|
("Dungeons & Dragons 5e", "dungeons and dragons"),
|
|
("Frankenstein Revised Edition", "frankenstein"),
|
|
("Dune Deluxe Edition Illustrated", "dune"),
|
|
("", ""),
|
|
(None, ""),
|
|
],
|
|
)
|
|
def test_titles_that_should_agree(self, title: str | None, expected: str) -> None:
|
|
assert normalize_title(title) == expected
|
|
|
|
def test_a_qualifier_that_is_the_title_survives(self) -> None:
|
|
"""A trailing qualifier is noise; the same word at the front is the book."""
|
|
assert normalize_title("The Illustrated Man") == "illustrated man"
|
|
|
|
def test_normalization_never_empties_a_title(self) -> None:
|
|
"""An article-only title is not improved by having no article left."""
|
|
assert normalize_title("The") == "the"
|
|
|
|
@pytest.mark.parametrize(
|
|
"title", ["Catch 22", "Fahrenheit 451", "Blade Runner 2049", "1984", "Apollo 13"]
|
|
)
|
|
def test_a_number_is_not_an_edition(self, title: str) -> None:
|
|
"""Edition stripping keys on the `e`; a bare number is part of the title."""
|
|
assert normalize_title(title) == title.casefold()
|
|
|
|
|
|
|
|
class TestNormalizeAuthor:
|
|
"""One person, written down several ways."""
|
|
|
|
@pytest.mark.parametrize(
|
|
("name", "expected"),
|
|
[
|
|
("Franz Kafka", "franz kafka"),
|
|
("Kafka, Franz", "franz kafka"),
|
|
("KAFKA, FRANZ", "franz kafka"),
|
|
("Émile Zola", "emile zola"),
|
|
("Doyle, Arthur Conan", "arthur conan doyle"),
|
|
# Runs of initials are joined, so spacing them out changes nothing.
|
|
("J.R.R. Tolkien", "jrr tolkien"),
|
|
("J. R. R. Tolkien", "jrr tolkien"),
|
|
("JRR Tolkien", "jrr tolkien"),
|
|
("", ""),
|
|
(None, ""),
|
|
],
|
|
)
|
|
def test_names_that_should_agree(self, name: str | None, expected: str) -> None:
|
|
assert normalize_author(name) == expected
|
|
|
|
def test_two_people_are_not_reduced_together(self) -> None:
|
|
"""Surname plus initial would collide unrelated writers; it is not used."""
|
|
assert normalize_author("Charles Dickens") != normalize_author("Colin Dexter")
|
|
|
|
def test_a_list_is_left_alone(self) -> None:
|
|
"""More than one comma is a list or a suffix, and guessing does more harm."""
|
|
assert normalize_author("Smith, John, Jr.") == "smith john jr"
|
|
|
|
|
|
class TestFormatAuthorName:
|
|
"""What gets stored and shown, as opposed to what gets compared."""
|
|
|
|
@pytest.mark.parametrize(
|
|
("written", "expected"),
|
|
[
|
|
# A leftover separator from a `DC:creator` list.
|
|
("Newman, Sam;", "Sam Newman"),
|
|
("Sam Newman ", "Sam Newman"),
|
|
(" Dan Vanderkam ", "Dan Vanderkam"),
|
|
# `Surname, Given` is how EPUBs file a name, not how anyone reads it.
|
|
("Kleppmann, Martin", "Martin Kleppmann"),
|
|
("Huxley, Aldous", "Aldous Huxley"),
|
|
("Liu, Cixin", "Cixin Liu"),
|
|
# An extension carried in from the filename the name was read out of.
|
|
("Sam Newman.epub", "Sam Newman"),
|
|
("Franz Kafka.mobi", "Franz Kafka"),
|
|
("Brian W. Kernighan.epub", "Brian W. Kernighan"),
|
|
("", ""),
|
|
(None, ""),
|
|
],
|
|
)
|
|
def test_names_are_tidied(self, written: str | None, expected: str) -> None:
|
|
assert format_author_name(written) == expected
|
|
|
|
@pytest.mark.parametrize(
|
|
"written",
|
|
[
|
|
# Two people in one string. Flipping it would invent a third.
|
|
"Dave Thomas, Andy Hunt",
|
|
"Mark Richards, Neal Ford",
|
|
# A compound surname is not recognised, and is left alone rather than
|
|
# rearranged wrongly.
|
|
"García Márquez, Gabriel",
|
|
],
|
|
)
|
|
def test_an_unrecognised_form_is_left_alone(self, written: str) -> None:
|
|
assert format_author_name(written) == written
|
|
|
|
def test_case_and_accents_belong_to_the_author(self) -> None:
|
|
"""Tidying removes what an extractor added; it does not correct spelling."""
|
|
assert format_author_name("Michał Płachta.epub") == "Michał Płachta"
|
|
assert format_author_name("Steve McConnell") == "Steve McConnell"
|
|
|
|
@pytest.mark.parametrize(
|
|
"written", ["Newman, Sam;", "Sam Newman.epub", "Kleppmann, Martin"]
|
|
)
|
|
def test_tidying_is_idempotent(self, written: str) -> None:
|
|
"""`unique_filter` tidies a name that may already be tidy; it must not drift."""
|
|
once = format_author_name(written)
|
|
assert format_author_name(once) == once
|
|
|
|
|
|
class TestNormalizeIdentifier:
|
|
"""Identifiers only help if the same edition produces the same key."""
|
|
|
|
def test_isbn_10_and_isbn_13_are_one_key(self) -> None:
|
|
assert normalize_identifier("isbn-10", "0486282112") == "isbn:9780486282114"
|
|
assert normalize_identifier("isbn-13", "9780486282114") == "isbn:9780486282114"
|
|
|
|
@pytest.mark.parametrize(
|
|
"written", ["978-0-486-28211-4", "978 0 486 28211 4", "9780486282114"]
|
|
)
|
|
def test_formatting_is_not_part_of_an_isbn(self, written: str) -> None:
|
|
assert normalize_identifier("isbn", written) == "isbn:9780486282114"
|
|
|
|
def test_an_isbn_that_fails_its_checksum_is_no_evidence(self) -> None:
|
|
assert normalize_identifier("isbn-13", "9780486282115") is None
|
|
|
|
def test_uuids_are_refused(self) -> None:
|
|
"""Generated per build, so they only re-find what the hash check catches."""
|
|
assert normalize_identifier("uuid", "3f2b1c4e-1111-2222-3333-444455556666") is None
|
|
assert normalize_identifier("urn:uuid", "3f2b1c4e-1111-2222-3333-444455556666") is None
|
|
|
|
def test_other_schemes_keep_their_own_key(self) -> None:
|
|
assert normalize_identifier("asin", "B000FC0PDA") == "asin:b000fc0pda"
|
|
assert normalize_identifier("ASIN", "b000fc0pda") == "asin:b000fc0pda"
|
|
|
|
def test_something_too_short_is_not_evidence(self) -> None:
|
|
"""A Calibre id of "42" would otherwise pair two unrelated books."""
|
|
assert normalize_identifier("calibre", "42") is None
|
|
|
|
@pytest.mark.parametrize(("name", "value"), [("", "1234567"), ("asin", "")])
|
|
def test_half_an_identifier_is_no_identifier(self, name: str, value: str) -> None:
|
|
assert normalize_identifier(name, value) is None
|
|
|
|
|
|
class TestIsbnConversion:
|
|
def test_isbn_10_converts_to_its_isbn_13(self) -> None:
|
|
assert isbn10_to_isbn13("0486282112") == "9780486282114"
|
|
|
|
def test_a_trailing_x_is_a_digit(self) -> None:
|
|
assert isbn10_to_isbn13("043942089X") == "9780439420891"
|
|
|
|
@pytest.mark.parametrize("isbn", ["0486282113", "9780486282114", "nonsense"])
|
|
def test_anything_that_is_not_an_isbn_10_converts_to_nothing(self, isbn: str) -> None:
|
|
assert isbn10_to_isbn13(isbn) is None
|
|
|
|
|
|
class TestParseIdentifier:
|
|
"""What an EPUB writes, and what is worth storing for it."""
|
|
|
|
@pytest.mark.parametrize(
|
|
"written",
|
|
[
|
|
"9780486282114",
|
|
"978-0-486-28211-4",
|
|
"urn:isbn:9780486282114",
|
|
"urn:isbn:978-0-486-28211-4",
|
|
"ISBN:978-0-486-28211-4",
|
|
],
|
|
)
|
|
def test_isbns_survive_however_they_are_written(self, written: str) -> None:
|
|
assert parse_identifier(written) == ("isbn-13", "9780486282114")
|
|
|
|
def test_the_scheme_attribute_is_read_too(self) -> None:
|
|
assert parse_identifier("0-486-28211-2", "ISBN") == ("isbn-10", "0486282112")
|
|
|
|
def test_non_isbn_identifiers_are_kept(self) -> None:
|
|
assert parse_identifier("urn:uuid:3f2b1c4e-1111-2222-3333-444455556666") == (
|
|
"uuid",
|
|
"3f2b1c4e-1111-2222-3333-444455556666",
|
|
)
|
|
assert parse_identifier("calibre:1234") == ("calibre", "1234")
|
|
assert parse_identifier("B000FC0PDA", "mobi-asin") == ("asin", "B000FC0PDA")
|
|
|
|
def test_an_unrecognised_prefix_is_part_of_the_value(self) -> None:
|
|
""""http://example.com/book" is not an identifier called "http"."""
|
|
assert parse_identifier("http://www.gutenberg.org/5200") == (
|
|
"id",
|
|
"http://www.gutenberg.org/5200",
|
|
)
|
|
|
|
def test_a_declared_isbn_that_is_not_one_is_dropped(self) -> None:
|
|
assert parse_identifier("urn:isbn:not-an-isbn") is None
|
|
|
|
@pytest.mark.parametrize("written", ["", " ", None])
|
|
def test_nothing_yields_nothing(self, written: str | None) -> None:
|
|
assert parse_identifier(written) is None
|