chemenu/tools/chemenu/tests/test_raw_derive_testfiles.py

"""`wikitool raw derive` against the real documents in `testfiles/`.

The synthetic fixtures in `binary_formats.py` and `test_raw_derive.py` prove the filters on
documents built to be easy; the files in `testfiles/` are whatever the world produced. Every file
there is derived, compared with itself on a second run and accepted into `raw/`, and an optional
`<file>.expect` next to it names the text that must come out. A distributed instance has no
`testfiles/`, so the whole module is then skipped.
"""
from __future__ import annotations

import importlib.util
import shutil
from pathlib import Path

import pytest

from chemenu import filters
from chemenu.commands.raw.accept import raw_accept_command
from chemenu.tests.test_raw_derive import (  # noqa: F401  (tree is a fixture)
    _derive,
    _header_and_text,
    tree,
)

TESTFILES = Path(__file__).resolve().parents[3] / "testfiles"
SIDECAR = ".expect"

pytestmark = pytest.mark.skipif(
    importlib.util.find_spec("pypdf") is None or importlib.util.find_spec("olefile") is None,
    reason="pypdf and olefile are needed for real PDFs, MSGs and archives holding them",
)


def _documents() -> list[Path]:
    if not TESTFILES.is_dir():
        return []
    return sorted(
        path
        for path in TESTFILES.rglob("*")
        if path.is_file()
        and not path.name.startswith(".")
        and path.name != "README.md"
        and path.suffix != SIDECAR
    )


def _id(path: Path) -> str:
    return path.relative_to(TESTFILES).as_posix()


def _texts(folder: Path) -> dict[str, str]:
    texts = {}
    for md in sorted(folder.rglob("*.md")):
        lines = md.read_text(encoding="utf-8").splitlines()
        texts[md.relative_to(folder).as_posix()] = "\n".join(
            line for line in lines if not line.startswith("derived:")
        )
    for tsv in sorted(folder.rglob("*.sheets/*.tsv")):
        texts[tsv.relative_to(folder).as_posix()] = tsv.read_text(encoding="utf-8")
    return texts


def _expected(document: Path) -> list[str]:
    sidecar = document.with_name(document.name + SIDECAR)
    if not sidecar.is_file():
        return []
    lines = sidecar.read_text(encoding="utf-8").splitlines()
    return [line for line in lines if line.strip() and not line.startswith("#")]


def _put_and_derive(tree: Path, document: Path) -> Path:
    (tree / "incoming" / document.name).write_bytes(document.read_bytes())
    _derive(f"incoming/{document.name}")
    return tree / "incoming" / document.stem


@pytest.mark.parametrize("document", _documents(), ids=_id)
def test_a_real_document_derives_stably_and_is_accepted(tree, document):
    assert filters.filter_for(document.name) is not None, (
        f"no import filter takes {document.name}; a file in testfiles/ must be one `raw derive` reads"
    )

    folder = _put_and_derive(tree, document)
    assert not (tree / "incoming" / document.name).exists()

    if document.suffix.lower() == ".zip":
        assert any(folder.rglob("*.md")), "an unpacked archive gets a .md beside its documents"
    else:
        original = folder / f"original{document.suffix.lower()}"
        assert original.read_bytes() == document.read_bytes()
        header, _text = _header_and_text(folder / "derived.md")
        assert header["derived_by"] == "wikitool raw derive"
        assert header["original_name"] == document.name

    first = _texts(folder)
    for needle in _expected(document):
        assert any(needle in text for text in first.values()), f"{needle!r} not in the derived text"

    shutil.rmtree(folder)
    assert _texts(_put_and_derive(tree, document)) == first, "the same bytes gave a different text"

    raw_accept_command(
        files=[Path(f"incoming/{document.stem}")], fidelity="verbatim", authority="reporting",
        page=None, replaces=None, dry_run=False,
    )
    assert not (tree / "incoming" / document.stem).exists()


def test_texts_covers_the_sheet_tsvs_beside_the_markdown(tmp_path):
    (tmp_path / "x.sheets").mkdir()
    (tmp_path / "derived.md").write_text("derived: now\nbody\n", encoding="utf-8")
    cells = tmp_path / "x.sheets" / "01-a.tsv"
    cells.write_text("a\tb\n", encoding="utf-8")

    before = _texts(tmp_path)
    assert before["derived.md"] == "body"
    assert before["x.sheets/01-a.tsv"] == "a\tb\n"

    cells.write_text("a\tc\n", encoding="utf-8")
    assert _texts(tmp_path) != before