feat: import a Calibre library
Reads metadata.db and copies the books into a library — from a zip uploaded on the library settings page, or from a path with `litestar calibre-import`. The source is never touched, and re-running only picks up what is new. Also names the formats mimetypes does not know: a Calibre library is full of MOBI and AZW3, and a null content type used to fail the book endpoint.
This commit is contained in:
@@ -0,0 +1,392 @@
|
||||
import zipfile
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from chitai.services.calibre import (
|
||||
CalibreLibrary,
|
||||
CalibreLibraryError,
|
||||
extract_calibre_archive,
|
||||
format_series_index,
|
||||
parse_date,
|
||||
strip_html,
|
||||
unescape_author,
|
||||
)
|
||||
|
||||
from tests.calibre_fixtures import UNDEFINED_DATE, CalibreFixture
|
||||
|
||||
|
||||
EPUB = Path("tests/data_files/Metamorphosis - Franz Kafka.epub")
|
||||
PDF = Path("tests/data_files/Calculus Made Easy - Silvanus Thompson.pdf")
|
||||
|
||||
|
||||
@pytest.fixture(name="library_root")
|
||||
def fx_library_root(tmp_path: Path) -> Path:
|
||||
"""A small Calibre library covering the rows that are easy to read wrongly."""
|
||||
fixture = CalibreFixture(tmp_path / "Calibre Library")
|
||||
fixture.add_pages_table()
|
||||
|
||||
fixture.add_book(
|
||||
1,
|
||||
"The Metamorphosis",
|
||||
authors=["Franz Kafka"],
|
||||
pubdate="1915-10-15 00:00:00+00:00",
|
||||
tags=["Fiction", "Absurdist"],
|
||||
publisher="Kurt Wolff Verlag",
|
||||
languages=["deu", "eng"],
|
||||
comment="<p>A travelling salesman.</p><p>He wakes up <i>changed</i>.</p>",
|
||||
identifiers={"isbn": "978-0-486-29030-0", "amazon": "B01N5IB20Q"},
|
||||
uuid="11111111-2222-3333-4444-555555555555",
|
||||
pages=201,
|
||||
cover=True,
|
||||
formats={"EPUB": EPUB},
|
||||
)
|
||||
|
||||
# Volume seven of a series, and no publication date — the two values most likely to
|
||||
# be carried through verbatim when they should not be.
|
||||
fixture.add_book(
|
||||
2,
|
||||
"Persepolis Rising",
|
||||
# Calibre escapes the comma and nothing else, so the space after it is stored
|
||||
# as-is: `Corey, Jr.` is written `Corey| Jr.`.
|
||||
authors=["Corey| Jr., James S. A."],
|
||||
series="The Expanse",
|
||||
series_index=7.0,
|
||||
formats={"EPUB": EPUB},
|
||||
)
|
||||
|
||||
# A novella between two novels: a fractional position is real and must survive.
|
||||
fixture.add_book(
|
||||
3, "Strange Dogs", series="The Expanse", series_index=6.5, formats={"PDF": PDF}
|
||||
)
|
||||
|
||||
# Every row Calibre will happily hold and Chitai cannot use: no files at all.
|
||||
fixture.add_book(4, "Metadata Only")
|
||||
|
||||
# A catalogue row whose file is not on disk.
|
||||
fixture.add_book(5, "Lost Book")
|
||||
fixture.add_missing_format(5, "EPUB", "Lost Book - Unknown")
|
||||
|
||||
return fixture.commit()
|
||||
|
||||
|
||||
async def test_reads_a_book_whole(library_root: Path) -> None:
|
||||
async with CalibreLibrary(library_root) as library:
|
||||
assert await library.count() == 5
|
||||
books = await library.books()
|
||||
|
||||
book = books[0]
|
||||
|
||||
assert book.calibre_id == 1
|
||||
assert book.title == "The Metamorphosis"
|
||||
assert book.authors == ["Franz Kafka"]
|
||||
assert book.published_date == date(1915, 10, 15)
|
||||
assert book.tags == ["Absurdist", "Fiction"]
|
||||
assert book.publisher == "Kurt Wolff Verlag"
|
||||
assert book.pages == 201
|
||||
assert book.uuid == "11111111-2222-3333-4444-555555555555"
|
||||
|
||||
# One language, and the one Calibre put first.
|
||||
assert book.language == "deu"
|
||||
|
||||
# Reported as Calibre wrote them: folding `amazon` onto `asin` is the importer's
|
||||
# job, not the reader's.
|
||||
assert book.identifiers == {"isbn": "978-0-486-29030-0", "amazon": "B01N5IB20Q"}
|
||||
|
||||
assert book.cover is not None
|
||||
assert book.cover.is_file()
|
||||
|
||||
assert len(book.files) == 1
|
||||
assert book.files[0].format == "EPUB"
|
||||
assert book.files[0].path.is_file()
|
||||
# The stem is Calibre's, truncated and sanitised — never the title.
|
||||
assert book.files[0].path.name != f"{book.title}.epub"
|
||||
|
||||
|
||||
async def test_the_undefined_date_is_not_a_date(library_root: Path) -> None:
|
||||
"""`0101-01-01` parses fine, which is exactly the problem."""
|
||||
async with CalibreLibrary(library_root) as library:
|
||||
books = {book.calibre_id: book for book in await library.books()}
|
||||
|
||||
assert books[2].published_date is None
|
||||
|
||||
|
||||
async def test_series_position_is_a_plain_string(library_root: Path) -> None:
|
||||
async with CalibreLibrary(library_root) as library:
|
||||
books = {book.calibre_id: book for book in await library.books()}
|
||||
|
||||
assert books[2].series == "The Expanse"
|
||||
assert books[2].series_position == "7"
|
||||
|
||||
assert books[3].series_position == "6.5"
|
||||
|
||||
# `series_index` defaults to 1.0 for every book, so a position without a series
|
||||
# would invent a volume one out of nothing.
|
||||
assert books[1].series is None
|
||||
assert books[1].series_position is None
|
||||
|
||||
|
||||
async def test_author_commas_are_unescaped(library_root: Path) -> None:
|
||||
async with CalibreLibrary(library_root) as library:
|
||||
books = {book.calibre_id: book for book in await library.books()}
|
||||
|
||||
assert books[2].authors == ["Corey, Jr., James S. A."]
|
||||
|
||||
|
||||
async def test_comments_come_back_as_text(library_root: Path) -> None:
|
||||
async with CalibreLibrary(library_root) as library:
|
||||
books = {book.calibre_id: book for book in await library.books()}
|
||||
|
||||
assert books[1].description == "A travelling salesman.\nHe wakes up changed."
|
||||
assert books[2].description is None
|
||||
|
||||
|
||||
async def test_files_are_reported_whether_or_not_they_exist(library_root: Path) -> None:
|
||||
"""
|
||||
The reader says what the catalogue says. Whether the bytes are there is a question
|
||||
for whoever is about to copy them, which stats them anyway.
|
||||
"""
|
||||
async with CalibreLibrary(library_root) as library:
|
||||
books = {book.calibre_id: book for book in await library.books()}
|
||||
|
||||
assert books[4].files == []
|
||||
|
||||
assert len(books[5].files) == 1
|
||||
assert not books[5].files[0].path.exists()
|
||||
|
||||
|
||||
async def test_a_library_without_the_pages_table_still_reads(tmp_path: Path) -> None:
|
||||
"""`books_pages_link` is recent; an older library simply does not have it."""
|
||||
fixture = CalibreFixture(tmp_path / "Old Library")
|
||||
fixture.add_book(1, "Old Book", formats={"EPUB": EPUB})
|
||||
root = fixture.commit()
|
||||
|
||||
async with CalibreLibrary(root) as library:
|
||||
books = await library.books()
|
||||
|
||||
assert books[0].pages is None
|
||||
|
||||
|
||||
async def test_the_original_is_never_opened(library_root: Path) -> None:
|
||||
"""
|
||||
The catalogue is copied before it is read, and the copy goes away afterwards.
|
||||
|
||||
Calibre may be running and writing; this is what keeps a live library out of it.
|
||||
"""
|
||||
before = (library_root / "metadata.db").read_bytes()
|
||||
|
||||
library = CalibreLibrary(library_root)
|
||||
await library.open()
|
||||
workspace = library._workspace
|
||||
|
||||
assert workspace is not None and (workspace / "metadata.db").is_file()
|
||||
|
||||
await library.close()
|
||||
|
||||
assert not workspace.exists()
|
||||
assert (library_root / "metadata.db").read_bytes() == before
|
||||
|
||||
|
||||
async def test_closing_twice_is_harmless(library_root: Path) -> None:
|
||||
library = CalibreLibrary(library_root)
|
||||
await library.open()
|
||||
await library.close()
|
||||
await library.close()
|
||||
|
||||
|
||||
async def test_a_directory_that_is_not_a_calibre_library(tmp_path: Path) -> None:
|
||||
with pytest.raises(CalibreLibraryError, match="not a Calibre library"):
|
||||
await CalibreLibrary(tmp_path).open()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("stored", "expected"),
|
||||
[
|
||||
("2017-12-04 04:00:00+00:00", date(2017, 12, 4)),
|
||||
("2001-07-02 00:00:00+00:00", date(2001, 7, 2)),
|
||||
("1999-01-31", date(1999, 1, 31)),
|
||||
# Calibre's sentinel, and anything else implausibly early.
|
||||
(UNDEFINED_DATE, None),
|
||||
("0101-01-01", None),
|
||||
(None, None),
|
||||
("", None),
|
||||
("not a date", None),
|
||||
],
|
||||
)
|
||||
def test_parse_date(stored: str | None, expected: date | None) -> None:
|
||||
assert parse_date(stored) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("index", "expected"),
|
||||
[
|
||||
(7.0, "7"),
|
||||
(1.0, "1"),
|
||||
(6.5, "6.5"),
|
||||
(0.0, "0"),
|
||||
(12.25, "12.25"),
|
||||
(None, None),
|
||||
],
|
||||
)
|
||||
def test_format_series_index(index: float | None, expected: str | None) -> None:
|
||||
assert format_series_index(index) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("stored", "expected"),
|
||||
[
|
||||
("Doyle| Sir Arthur Conan", "Doyle, Sir Arthur Conan"),
|
||||
("Franz Kafka", "Franz Kafka"),
|
||||
(" Herman Melville ", "Herman Melville"),
|
||||
],
|
||||
)
|
||||
def test_unescape_author(stored: str, expected: str) -> None:
|
||||
assert unescape_author(stored) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("html", "expected"),
|
||||
[
|
||||
("<p>One.</p><p>Two.</p>", "One.\nTwo."),
|
||||
("Plain text", "Plain text"),
|
||||
("<div>A<br>B</div>", "A\nB"),
|
||||
("<p>Café & bar</p>", "Café & bar"),
|
||||
("<ul><li>One</li><li>Two</li></ul>", "One\nTwo"),
|
||||
# Markup carrying no text at all is nothing, not an empty description.
|
||||
("<p></p>", None),
|
||||
("", None),
|
||||
(None, None),
|
||||
],
|
||||
)
|
||||
def test_strip_html(html: str | None, expected: str | None) -> None:
|
||||
assert strip_html(html) == expected
|
||||
|
||||
|
||||
class TestArchives:
|
||||
"""A Calibre library that arrives zipped rather than as a path."""
|
||||
|
||||
def zipped(self, root: Path, into: Path, prefix: str = "") -> Path:
|
||||
"""Zip a directory the way a file manager would."""
|
||||
archive = into / "library.zip"
|
||||
|
||||
with zipfile.ZipFile(archive, "w") as writing:
|
||||
for path in sorted(root.rglob("*")):
|
||||
if path.is_file():
|
||||
writing.write(path, f"{prefix}{path.relative_to(root)}")
|
||||
|
||||
return archive
|
||||
|
||||
async def test_a_library_zipped_at_its_root(
|
||||
self, library_root: Path, tmp_path: Path
|
||||
) -> None:
|
||||
archive = self.zipped(library_root, tmp_path)
|
||||
destination = tmp_path / "unpacked"
|
||||
destination.mkdir()
|
||||
|
||||
catalogue = await extract_calibre_archive(archive, destination)
|
||||
|
||||
assert catalogue == destination
|
||||
async with CalibreLibrary(catalogue) as library:
|
||||
assert await library.count() == 5
|
||||
|
||||
async def test_a_library_zipped_inside_a_folder(
|
||||
self, library_root: Path, tmp_path: Path
|
||||
) -> None:
|
||||
"""Zipping the folder itself is at least as common as zipping its contents."""
|
||||
archive = self.zipped(library_root, tmp_path, prefix="Calibre Library/")
|
||||
destination = tmp_path / "unpacked"
|
||||
destination.mkdir()
|
||||
|
||||
catalogue = await extract_calibre_archive(archive, destination)
|
||||
|
||||
assert catalogue == destination / "Calibre Library"
|
||||
async with CalibreLibrary(catalogue) as library:
|
||||
assert await library.count() == 5
|
||||
|
||||
async def test_an_entry_pointing_outside_the_archive_is_refused(
|
||||
self, tmp_path: Path
|
||||
) -> None:
|
||||
"""
|
||||
Zip slip. `ZipFile.extract` sanitises names itself, but relying on that silently
|
||||
is how the next person to change the extraction call reintroduces it.
|
||||
"""
|
||||
archive = tmp_path / "hostile.zip"
|
||||
|
||||
with zipfile.ZipFile(archive, "w") as writing:
|
||||
writing.writestr("metadata.db", "not really")
|
||||
writing.writestr("../../escaped.txt", "gotcha")
|
||||
|
||||
destination = tmp_path / "unpacked"
|
||||
destination.mkdir()
|
||||
|
||||
with pytest.raises(CalibreLibraryError, match="outside itself"):
|
||||
await extract_calibre_archive(archive, destination)
|
||||
|
||||
assert not (tmp_path.parent / "escaped.txt").exists()
|
||||
|
||||
async def test_something_that_is_not_a_zip(self, tmp_path: Path) -> None:
|
||||
archive = tmp_path / "not.zip"
|
||||
archive.write_bytes(b"PK-ish, but no")
|
||||
|
||||
destination = tmp_path / "unpacked"
|
||||
destination.mkdir()
|
||||
|
||||
with pytest.raises(CalibreLibraryError, match="not a zip file"):
|
||||
await extract_calibre_archive(archive, destination)
|
||||
|
||||
async def test_an_archive_with_no_catalogue(self, tmp_path: Path) -> None:
|
||||
archive = tmp_path / "books.zip"
|
||||
|
||||
with zipfile.ZipFile(archive, "w") as writing:
|
||||
writing.writestr("Some Book.epub", "content")
|
||||
|
||||
destination = tmp_path / "unpacked"
|
||||
destination.mkdir()
|
||||
|
||||
with pytest.raises(CalibreLibraryError, match="no metadata.db"):
|
||||
await extract_calibre_archive(archive, destination)
|
||||
|
||||
# Refused before anything was written.
|
||||
assert list(destination.iterdir()) == []
|
||||
|
||||
async def test_a_catalogue_buried_too_deep(self, tmp_path: Path) -> None:
|
||||
"""Somebody's whole backup tree is not a library, however much it contains one."""
|
||||
archive = tmp_path / "backup.zip"
|
||||
|
||||
with zipfile.ZipFile(archive, "w") as writing:
|
||||
writing.writestr("backups/2026/january/library/metadata.db", "not really")
|
||||
|
||||
destination = tmp_path / "unpacked"
|
||||
destination.mkdir()
|
||||
|
||||
with pytest.raises(CalibreLibraryError, match="within 3 levels"):
|
||||
await extract_calibre_archive(archive, destination)
|
||||
|
||||
async def test_an_archive_too_big_for_the_disk(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""
|
||||
Checked before writing, not discovered part-way through.
|
||||
|
||||
A full disk takes the whole application down, and the size is in the archive
|
||||
already.
|
||||
"""
|
||||
archive = tmp_path / "huge.zip"
|
||||
|
||||
with zipfile.ZipFile(archive, "w") as writing:
|
||||
writing.writestr("metadata.db", "not really")
|
||||
|
||||
destination = tmp_path / "unpacked"
|
||||
destination.mkdir()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"chitai.services.calibre.shutil.disk_usage",
|
||||
lambda _path: SimpleNamespace(total=1024, used=1024, free=0),
|
||||
)
|
||||
|
||||
with pytest.raises(CalibreLibraryError, match="only"):
|
||||
await extract_calibre_archive(archive, destination)
|
||||
|
||||
assert list(destination.iterdir()) == []
|
||||
@@ -0,0 +1,64 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from chitai.services.utils import guess_content_type
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "expected"),
|
||||
[
|
||||
# What `mimetypes` already knows, kept here so a host with a thin
|
||||
# /etc/mime.types cannot change the answer without a test noticing.
|
||||
("Frankenstein.epub", "application/epub+zip"),
|
||||
("Calculus.pdf", "application/pdf"),
|
||||
("Persepolis.azw3", "application/vnd.amazon.mobi8-ebook"),
|
||||
("Watchmen.cbz", "application/vnd.comicbook+zip"),
|
||||
# What it does not, and where a Calibre library's older formats live.
|
||||
("Dune.mobi", "application/x-mobipocket-ebook"),
|
||||
("Dune.prc", "application/x-mobipocket-ebook"),
|
||||
("Dune.azw", "application/vnd.amazon.ebook"),
|
||||
("Voyna i Mir.fb2", "application/x-fictionbook+xml"),
|
||||
("Voyna i Mir.fbz", "application/x-zip-compressed-fb2"),
|
||||
("Reader.lit", "application/x-ms-reader"),
|
||||
("Reader.lrf", "application/x-sony-bbeb"),
|
||||
("Watchmen.cb7", "application/x-cb7"),
|
||||
# Case is not part of the answer, and Calibre writes formats uppercase.
|
||||
("Dune.MOBI", "application/x-mobipocket-ebook"),
|
||||
# Nothing can name these, and None is the answer rather than a placeholder.
|
||||
("Notes.xyzzy", None),
|
||||
("README", None),
|
||||
],
|
||||
)
|
||||
def test_guess_content_type(filename: str, expected: str | None) -> None:
|
||||
assert guess_content_type(Path(filename)) == expected
|
||||
# A str and a Path must agree, and an upload's `filename` carries its relative
|
||||
# path, so a name with directories in front of it has to resolve the same way.
|
||||
assert guess_content_type(filename) == expected
|
||||
assert guess_content_type(f"Some Author/Some Book/{filename}") == expected
|
||||
|
||||
|
||||
def test_fallback_is_used_only_when_the_extension_says_nothing() -> None:
|
||||
"""A client's claim fills a gap; it never overrides the name."""
|
||||
assert (
|
||||
guess_content_type(Path("Dune.mobi"), fallback="application/pdf")
|
||||
== "application/x-mobipocket-ebook"
|
||||
)
|
||||
assert (
|
||||
guess_content_type(Path("Notes.xyzzy"), fallback="application/epub+zip")
|
||||
== "application/epub+zip"
|
||||
)
|
||||
|
||||
|
||||
def test_an_unspecified_fallback_is_not_an_answer() -> None:
|
||||
"""
|
||||
`application/octet-stream` from a client is it saying it does not know.
|
||||
|
||||
Browsers post exactly that for every extension they do not recognise, which is most
|
||||
ebook formats. Storing it would be indistinguishable from having determined a
|
||||
format, so it is discarded and the column keeps its null.
|
||||
"""
|
||||
assert (
|
||||
guess_content_type(Path("Notes.xyzzy"), fallback="application/octet-stream")
|
||||
is None
|
||||
)
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Tests for BookPathGenerator."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from chitai.services.filesystem_library import BookPathGenerator, sanitize_path_component
|
||||
|
||||
|
||||
ROOT = Path("/library")
|
||||
|
||||
|
||||
def path_for(**book) -> Path:
|
||||
return BookPathGenerator(ROOT).generate_path(book)
|
||||
|
||||
|
||||
def test_author_and_title() -> None:
|
||||
assert path_for(title="Dune", authors=["Frank Herbert"]) == (
|
||||
ROOT / "Frank Herbert" / "Dune"
|
||||
)
|
||||
|
||||
|
||||
def test_a_book_with_no_authors() -> None:
|
||||
assert path_for(title="Beowulf", authors=[]) == ROOT / "Unknown" / "Beowulf"
|
||||
|
||||
|
||||
def test_a_series_adds_a_level_and_pads_the_position() -> None:
|
||||
assert path_for(
|
||||
title="Persepolis Rising",
|
||||
authors=["James S. A. Corey"],
|
||||
series="The Expanse",
|
||||
series_position="7",
|
||||
) == ROOT / "James S. A. Corey" / "The Expanse" / "07 - Persepolis Rising"
|
||||
|
||||
|
||||
def test_a_slash_in_a_title_does_not_add_a_directory() -> None:
|
||||
"""
|
||||
The separators in the path come from the template, never from the metadata.
|
||||
|
||||
A title with a slash in it — "AC/DC", "Him/Her" — would otherwise put the book one
|
||||
level below where `book.path` says it is, which is what deletes, moves and file
|
||||
lookups all act on. Calibre keeps the real title in its database and strips this
|
||||
from its own directory names, so an import is where they surface.
|
||||
"""
|
||||
generated = path_for(title="Back in Black: AC/DC", authors=["Murray Engleheart"])
|
||||
|
||||
assert generated == ROOT / "Murray Engleheart" / "Back in Black: AC_DC"
|
||||
assert generated.relative_to(ROOT).parts == ("Murray Engleheart", "Back in Black: AC_DC")
|
||||
|
||||
|
||||
def test_a_slash_in_an_author_or_series_is_handled_too() -> None:
|
||||
assert path_for(title="Split", authors=["A/B Collective"]) == (
|
||||
ROOT / "A_B Collective" / "Split"
|
||||
)
|
||||
assert path_for(
|
||||
title="Volume One", authors=["Someone"], series="Either/Or", series_position="1"
|
||||
) == ROOT / "Someone" / "Either_Or" / "01 - Volume One"
|
||||
|
||||
|
||||
def test_control_characters_are_removed() -> None:
|
||||
assert path_for(title="Line\nBreak", authors=["Someone"]) == (
|
||||
ROOT / "Someone" / "Line_Break"
|
||||
)
|
||||
|
||||
|
||||
def test_sanitize_path_component() -> None:
|
||||
assert sanitize_path_component("AC/DC") == "AC_DC"
|
||||
assert sanitize_path_component("back\\slash") == "back_slash"
|
||||
assert sanitize_path_component(" padded ") == "padded"
|
||||
# Colons and other punctuation are legal in a path and are left alone.
|
||||
assert sanitize_path_component("Title: Subtitle") == "Title: Subtitle"
|
||||
@@ -0,0 +1,415 @@
|
||||
"""Tests for importing a Calibre library through BookService."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from chitai.config import settings
|
||||
from chitai.database import models as m
|
||||
from chitai.services import BookService
|
||||
from chitai.services.calibre import CalibreLibrary
|
||||
|
||||
from tests.calibre_fixtures import CalibreFixture
|
||||
|
||||
|
||||
DATA_FILES = Path("tests/data_files")
|
||||
EPUB = DATA_FILES / "Metamorphosis - Franz Kafka.epub"
|
||||
OTHER_EPUB = DATA_FILES / "The Art of War - Sun Tzu.epub"
|
||||
PDF = DATA_FILES / "Calculus Made Easy - Silvanus Thompson.pdf"
|
||||
|
||||
|
||||
@pytest.fixture(name="calibre_root")
|
||||
def fx_calibre_root(tmp_path: Path) -> Path:
|
||||
"""Three books: one plain, one in two formats, one Chitai cannot use."""
|
||||
fixture = CalibreFixture(tmp_path / "source")
|
||||
fixture.add_pages_table()
|
||||
|
||||
fixture.add_book(
|
||||
1,
|
||||
"The Metamorphosis",
|
||||
authors=["Franz Kafka"],
|
||||
pubdate="1915-10-15 00:00:00+00:00",
|
||||
tags=["Fiction", "Absurdist"],
|
||||
publisher="Kurt Wolff Verlag",
|
||||
languages=["deu"],
|
||||
comment="<p>He wakes up <i>changed</i>.</p>",
|
||||
identifiers={"isbn": "978-0-486-29030-0", "amazon": "B01N5IB20Q"},
|
||||
uuid="11111111-2222-3333-4444-555555555555",
|
||||
pages=201,
|
||||
cover=True,
|
||||
formats={"EPUB": EPUB},
|
||||
)
|
||||
|
||||
fixture.add_book(
|
||||
2,
|
||||
"The Art of War",
|
||||
authors=["Sun Tzu"],
|
||||
series="Classics",
|
||||
series_index=3.0,
|
||||
formats={"EPUB": OTHER_EPUB, "PDF": PDF},
|
||||
)
|
||||
|
||||
fixture.add_book(3, "Metadata Only", authors=["Nobody"])
|
||||
|
||||
return fixture.commit()
|
||||
|
||||
|
||||
async def library_of(root: Path) -> CalibreLibrary:
|
||||
source = CalibreLibrary(root)
|
||||
await source.open()
|
||||
return source
|
||||
|
||||
|
||||
async def test_imports_a_catalogue(
|
||||
books_service: BookService, test_library: m.Library, calibre_root: Path
|
||||
) -> None:
|
||||
source = await library_of(calibre_root)
|
||||
try:
|
||||
result = await books_service.create_many_from_calibre(source, test_library)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
assert result.total == 3
|
||||
assert len(result.created) == 2
|
||||
|
||||
# The book with no files is left out: a record with nothing to read, and a directory
|
||||
# to match, is worse than not importing it.
|
||||
assert [skipped.calibre_id for skipped in result.skipped] == [3]
|
||||
assert result.skipped[0].reason == "no files in the catalogue"
|
||||
assert result.failed == []
|
||||
|
||||
book = await books_service.get(result.created[0])
|
||||
|
||||
assert book.title == "The Metamorphosis"
|
||||
assert [author.name for author in book.authors] == ["Franz Kafka"]
|
||||
assert sorted(tag.name for tag in book.tags) == ["Absurdist", "Fiction"]
|
||||
assert book.publisher is not None and book.publisher.name == "Kurt Wolff Verlag"
|
||||
assert book.published_date is not None and book.published_date.year == 1915
|
||||
assert book.language == "deu"
|
||||
assert book.pages == 201
|
||||
assert book.cover_image is not None
|
||||
|
||||
# The HTML is gone; `Book.description` is rendered as text.
|
||||
assert book.description == "He wakes up changed."
|
||||
|
||||
|
||||
async def test_two_formats_are_one_book(
|
||||
books_service: BookService, test_library: m.Library, calibre_root: Path
|
||||
) -> None:
|
||||
source = await library_of(calibre_root)
|
||||
try:
|
||||
result = await books_service.create_many_from_calibre(source, test_library)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
book = await books_service.get(result.created[1])
|
||||
|
||||
assert book.title == "The Art of War"
|
||||
assert sorted(Path(file.path).suffix for file in book.files) == [".epub", ".pdf"]
|
||||
|
||||
# A REAL series index reaches the column as the string everything else writes.
|
||||
assert book.series is not None and book.series.title == "Classics"
|
||||
assert book.series_position == "3"
|
||||
|
||||
|
||||
async def test_identifiers_are_folded_onto_chitai_schemes(
|
||||
books_service: BookService, test_library: m.Library, calibre_root: Path
|
||||
) -> None:
|
||||
"""
|
||||
`amazon` becomes `asin`, a hyphenated ISBN survives, and the Calibre uuid is kept.
|
||||
|
||||
The uuid is deliberately not stored under `uuid`, which duplicate matching ignores
|
||||
because an EPUB regenerates one per build. Calibre's is stable, so it is the durable
|
||||
link back to the row it came from.
|
||||
"""
|
||||
source = await library_of(calibre_root)
|
||||
try:
|
||||
result = await books_service.create_many_from_calibre(source, test_library)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
book = await books_service.get(result.created[0])
|
||||
identifiers = {identifier.name: identifier.value for identifier in book.identifiers}
|
||||
|
||||
assert identifiers["asin"] == "B01N5IB20Q"
|
||||
assert identifiers["isbn-13"] == "9780486290300"
|
||||
assert identifiers["calibre-uuid"] == "11111111-2222-3333-4444-555555555555"
|
||||
|
||||
matching = {
|
||||
identifier.name: identifier.normalized_value for identifier in book.identifiers
|
||||
}
|
||||
|
||||
# Stored under its own name, matched under one scheme for both ISBN forms.
|
||||
assert matching["isbn-13"] == "isbn:9780486290300"
|
||||
|
||||
# And the uuid carries a real matching key, which is the whole reason it is not
|
||||
# filed under `uuid`.
|
||||
assert matching["calibre-uuid"] is not None
|
||||
|
||||
|
||||
async def test_the_source_library_is_left_alone(
|
||||
books_service: BookService, test_library: m.Library, calibre_root: Path
|
||||
) -> None:
|
||||
"""Files are copied. Moving them would leave `metadata.db` pointing at nothing."""
|
||||
before = {
|
||||
path: path.stat().st_mtime_ns
|
||||
for path in sorted(calibre_root.rglob("*"))
|
||||
if path.is_file()
|
||||
}
|
||||
|
||||
source = await library_of(calibre_root)
|
||||
try:
|
||||
result = await books_service.create_many_from_calibre(source, test_library)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
after = {
|
||||
path: path.stat().st_mtime_ns
|
||||
for path in sorted(calibre_root.rglob("*"))
|
||||
if path.is_file()
|
||||
}
|
||||
|
||||
assert after == before
|
||||
|
||||
# And the copies are really there, under the library's own layout.
|
||||
for book_id in result.created:
|
||||
book = await books_service.get(book_id)
|
||||
for file in book.files:
|
||||
assert (Path(book.path or "") / file.path).is_file()
|
||||
|
||||
|
||||
async def test_importing_twice_creates_nothing(
|
||||
books_service: BookService, test_library: m.Library, calibre_root: Path
|
||||
) -> None:
|
||||
"""
|
||||
Re-running is safe with no bookkeeping: the bytes are recognised wherever they sit.
|
||||
|
||||
This is what makes an interrupted import resumable by simply running it again.
|
||||
"""
|
||||
for _ in range(2):
|
||||
source = await library_of(calibre_root)
|
||||
try:
|
||||
result = await books_service.create_many_from_calibre(source, test_library)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
assert result.created == []
|
||||
assert sorted(skipped.reason for skipped in result.skipped) == [
|
||||
"already stored",
|
||||
"already stored",
|
||||
"no files in the catalogue",
|
||||
]
|
||||
|
||||
held_by = [
|
||||
skipped.book_id for skipped in result.skipped if skipped.reason == "already stored"
|
||||
]
|
||||
assert all(book_id is not None for book_id in held_by)
|
||||
|
||||
|
||||
async def test_a_file_the_catalogue_lists_but_disk_does_not(
|
||||
books_service: BookService, test_library: m.Library, tmp_path: Path
|
||||
) -> None:
|
||||
"""Calibre keeps the row when a file is moved away behind its back."""
|
||||
fixture = CalibreFixture(tmp_path / "source")
|
||||
fixture.add_book(1, "Present", authors=["A"], formats={"EPUB": EPUB})
|
||||
fixture.add_book(2, "Absent", authors=["B"])
|
||||
fixture.add_missing_format(2, "EPUB", "Absent - B")
|
||||
root = fixture.commit()
|
||||
|
||||
source = await library_of(root)
|
||||
try:
|
||||
result = await books_service.create_many_from_calibre(source, test_library)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
assert len(result.created) == 1
|
||||
assert [(s.calibre_id, s.reason) for s in result.skipped] == [(2, "no files on disk")]
|
||||
|
||||
|
||||
async def test_one_broken_book_does_not_stop_the_import(
|
||||
books_service: BookService, test_library: m.Library, tmp_path: Path
|
||||
) -> None:
|
||||
"""
|
||||
A failure is recorded and the run continues, leaving no files behind for it.
|
||||
|
||||
An orphaned directory would make the next attempt reserve `title (2)` and look as
|
||||
though it had worked.
|
||||
"""
|
||||
fixture = CalibreFixture(tmp_path / "source")
|
||||
fixture.add_book(1, "First", authors=["A"], formats={"EPUB": EPUB})
|
||||
fixture.add_book(2, "Doomed", authors=["B"], formats={"EPUB": OTHER_EPUB})
|
||||
fixture.add_book(3, "Third", authors=["C"], formats={"PDF": PDF})
|
||||
root = fixture.commit()
|
||||
|
||||
original = books_service.create
|
||||
|
||||
async def fail_on_the_second(data, *args, **kwargs):
|
||||
if isinstance(data, dict) and data.get("title") == "Doomed":
|
||||
raise RuntimeError("no room on the shelf")
|
||||
return await original(data, *args, **kwargs)
|
||||
|
||||
books_service.create = fail_on_the_second # type: ignore[method-assign]
|
||||
|
||||
source = await library_of(root)
|
||||
try:
|
||||
result = await books_service.create_many_from_calibre(source, test_library)
|
||||
finally:
|
||||
await source.close()
|
||||
books_service.create = original # type: ignore[method-assign]
|
||||
|
||||
assert len(result.created) == 2
|
||||
assert len(result.failed) == 1
|
||||
assert result.failed[0].calibre_id == 2
|
||||
assert "no room on the shelf" in result.failed[0].reason
|
||||
|
||||
# Nothing of the failed book was left in the library. Checked against the path the
|
||||
# template would have produced, rather than by walking the root — the Calibre source
|
||||
# sits under it in these tests, and its own files are meant to still be there.
|
||||
assert not (Path(test_library.root_path) / "B").exists()
|
||||
|
||||
# And the books either side of it are where they should be.
|
||||
for book_id in result.created:
|
||||
book = await books_service.get(book_id)
|
||||
for file in book.files:
|
||||
assert (Path(book.path or "") / file.path).is_file()
|
||||
|
||||
|
||||
async def test_a_cover_that_cannot_be_read_is_not_fatal(
|
||||
books_service: BookService, test_library: m.Library, tmp_path: Path
|
||||
) -> None:
|
||||
"""
|
||||
A truncated `cover.jpg` costs the cover, not the book.
|
||||
|
||||
Real libraries hold them, from an interrupted download or a failed conversion, and
|
||||
the cover is the one thing in the directory that can be replaced from the book page.
|
||||
"""
|
||||
fixture = CalibreFixture(tmp_path / "source")
|
||||
fixture.add_book(
|
||||
1, "Unreadable Cover", authors=["A"], corrupt_cover=True, formats={"EPUB": EPUB}
|
||||
)
|
||||
root = fixture.commit()
|
||||
|
||||
source = await library_of(root)
|
||||
try:
|
||||
result = await books_service.create_many_from_calibre(source, test_library)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
assert result.failed == []
|
||||
assert len(result.created) == 1
|
||||
|
||||
book = await books_service.get(result.created[0])
|
||||
assert book.cover_image is None
|
||||
assert len(book.files) == 1
|
||||
|
||||
|
||||
async def test_shared_authors_and_tags_are_one_row_each(
|
||||
books_service: BookService, test_library: m.Library, tmp_path: Path, session
|
||||
) -> None:
|
||||
"""Two books by one author must not produce two `Author` rows."""
|
||||
fixture = CalibreFixture(tmp_path / "source")
|
||||
fixture.add_book(
|
||||
1, "One", authors=["Franz Kafka"], tags=["Fiction"], formats={"EPUB": EPUB}
|
||||
)
|
||||
fixture.add_book(
|
||||
2, "Two", authors=["Franz Kafka"], tags=["Fiction"], formats={"EPUB": OTHER_EPUB}
|
||||
)
|
||||
root = fixture.commit()
|
||||
|
||||
source = await library_of(root)
|
||||
try:
|
||||
result = await books_service.create_many_from_calibre(source, test_library)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
assert len(result.created) == 2
|
||||
|
||||
first, second = [await books_service.get(book_id) for book_id in result.created]
|
||||
|
||||
assert first.authors[0].id == second.authors[0].id
|
||||
assert first.tags[0].id == second.tags[0].id
|
||||
|
||||
|
||||
async def test_a_second_copy_is_reported_not_refused(
|
||||
books_service: BookService, test_library: m.Library, tmp_path: Path
|
||||
) -> None:
|
||||
"""
|
||||
Two catalogue rows for one book, with different bytes, both import.
|
||||
|
||||
File-level dedupe cannot see it — the archives differ — so book-level detection
|
||||
reports the pair and leaves the decision to the reader.
|
||||
"""
|
||||
padded = tmp_path / "padded.epub"
|
||||
padded.write_bytes(EPUB.read_bytes() + b"\0" * 64)
|
||||
|
||||
fixture = CalibreFixture(tmp_path / "source")
|
||||
fixture.add_book(
|
||||
1, "The Metamorphosis", authors=["Franz Kafka"], formats={"EPUB": EPUB}
|
||||
)
|
||||
fixture.add_book(
|
||||
2, "The Metamorphosis", authors=["Franz Kafka"], formats={"EPUB": padded}
|
||||
)
|
||||
root = fixture.commit()
|
||||
|
||||
source = await library_of(root)
|
||||
try:
|
||||
result = await books_service.create_many_from_calibre(source, test_library)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
assert len(result.created) == 2
|
||||
assert len(result.possible_duplicates) == 1
|
||||
assert result.possible_duplicates[0].candidates[0].book_id == result.created[0]
|
||||
|
||||
|
||||
async def test_allow_duplicates_stores_the_same_bytes_again(
|
||||
books_service: BookService, test_library: m.Library, calibre_root: Path
|
||||
) -> None:
|
||||
for allow in (False, True):
|
||||
source = await library_of(calibre_root)
|
||||
try:
|
||||
result = await books_service.create_many_from_calibre(
|
||||
source, test_library, allow_duplicates=allow
|
||||
)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
assert len(result.created) == 2
|
||||
|
||||
|
||||
async def test_duplicate_scope_off_imports_everything(
|
||||
books_service: BookService,
|
||||
test_library: m.Library,
|
||||
calibre_root: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(settings, "duplicate_scope", "off")
|
||||
|
||||
for _ in range(2):
|
||||
source = await library_of(calibre_root)
|
||||
try:
|
||||
result = await books_service.create_many_from_calibre(source, test_library)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
assert len(result.created) == 2
|
||||
|
||||
|
||||
async def test_progress_is_reported_per_book(
|
||||
books_service: BookService, test_library: m.Library, calibre_root: Path
|
||||
) -> None:
|
||||
"""The import is long enough that its progress is the only thing worth watching."""
|
||||
seen = []
|
||||
|
||||
source = await library_of(calibre_root)
|
||||
try:
|
||||
await books_service.create_many_from_calibre(
|
||||
source, test_library, on_progress=seen.append
|
||||
)
|
||||
finally:
|
||||
await source.close()
|
||||
|
||||
assert [progress.processed for progress in seen] == [1, 2, 3]
|
||||
assert all(progress.total == 3 for progress in seen)
|
||||
assert [progress.outcome for progress in seen] == ["created", "created", "skipped"]
|
||||
assert seen[0].title == "The Metamorphosis"
|
||||
Reference in New Issue
Block a user