torbenandClaude Code
9868fce3d5
Each sheet becomes NN-<sheet>.tsv (RFC 4180, tab-delimited, A1-aligned) in a <stem>.sheets/ folder beside the .md; sheets with formulas also get a .formulas.tsv. derived.md is the overview. Adds testfiles/ with a CI test. Files changed: - .gitattributes - CHANGES.md - VERSION - docs/dev/development.md - docs/how-to/ingest-a-source.md - docs/reference/commands/raw-material-and-uploads.md - raw/CONTRACT.md - testfiles/README.md - tools/CONTRACT.md - tools/chemenu/commands/raw_cmd.py - tools/chemenu/filters/__init__.py - tools/chemenu/filters/archive.py - tools/chemenu/filters/ooxml.py - tools/chemenu/tests/test_raw_derive.py - tools/chemenu/tests/test_raw_derive_testfiles.py Co-Authored-By: Claude Code <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
105 lines
3.7 KiB
Python
105 lines
3.7 KiB
Python
"""`wikitool raw derive` against the real documents in `testfiles/`.
|
|
|
|
The synthetic fixtures in `binary_formats.py` and `test_raw_derive.py` prove the filters on
|
|
documents built to be easy; the files in `testfiles/` are whatever the world produced. Every file
|
|
there is derived, compared with itself on a second run and accepted into `raw/`, and an optional
|
|
`<file>.expect` next to it names the text that must come out. A distributed instance has no
|
|
`testfiles/`, so the whole module is then skipped.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from chemenu import filters
|
|
from chemenu.commands.raw_cmd import raw_accept_command
|
|
from chemenu.tests.test_raw_derive import ( # noqa: F401 (tree is a fixture)
|
|
_derive,
|
|
_header_and_text,
|
|
tree,
|
|
)
|
|
|
|
TESTFILES = Path(__file__).resolve().parents[3] / "testfiles"
|
|
SIDECAR = ".expect"
|
|
|
|
pytestmark = pytest.mark.skipif(
|
|
importlib.util.find_spec("pypdf") is None or importlib.util.find_spec("olefile") is None,
|
|
reason="pypdf and olefile are needed for real PDFs, MSGs and archives holding them",
|
|
)
|
|
|
|
|
|
def _documents() -> list[Path]:
|
|
if not TESTFILES.is_dir():
|
|
return []
|
|
return sorted(
|
|
path
|
|
for path in TESTFILES.rglob("*")
|
|
if path.is_file()
|
|
and not path.name.startswith(".")
|
|
and path.name != "README.md"
|
|
and path.suffix != SIDECAR
|
|
)
|
|
|
|
|
|
def _id(path: Path) -> str:
|
|
return path.relative_to(TESTFILES).as_posix()
|
|
|
|
|
|
def _texts(folder: Path) -> dict[str, str]:
|
|
texts = {}
|
|
for md in sorted(folder.rglob("*.md")):
|
|
lines = md.read_text(encoding="utf-8").splitlines()
|
|
texts[md.relative_to(folder).as_posix()] = "\n".join(
|
|
line for line in lines if not line.startswith("derived:")
|
|
)
|
|
return texts
|
|
|
|
|
|
def _expected(document: Path) -> list[str]:
|
|
sidecar = document.with_name(document.name + SIDECAR)
|
|
if not sidecar.is_file():
|
|
return []
|
|
lines = sidecar.read_text(encoding="utf-8").splitlines()
|
|
return [line for line in lines if line.strip() and not line.startswith("#")]
|
|
|
|
|
|
def _put_and_derive(tree: Path, document: Path) -> Path:
|
|
(tree / "incoming" / document.name).write_bytes(document.read_bytes())
|
|
_derive(f"incoming/{document.name}")
|
|
return tree / "incoming" / document.stem
|
|
|
|
|
|
@pytest.mark.parametrize("document", _documents(), ids=_id)
|
|
def test_a_real_document_derives_stably_and_is_accepted(tree, document):
|
|
assert filters.filter_for(document.name) is not None, (
|
|
f"no import filter takes {document.name}; a file in testfiles/ must be one `raw derive` reads"
|
|
)
|
|
|
|
folder = _put_and_derive(tree, document)
|
|
assert not (tree / "incoming" / document.name).exists()
|
|
|
|
if document.suffix.lower() == ".zip":
|
|
assert any(folder.rglob("*.md")), "an unpacked archive gets a .md beside its documents"
|
|
else:
|
|
original = folder / f"original{document.suffix.lower()}"
|
|
assert original.read_bytes() == document.read_bytes()
|
|
header, _text = _header_and_text(folder / "derived.md")
|
|
assert header["derived_by"] == "wikitool raw derive"
|
|
assert header["original_name"] == document.name
|
|
|
|
first = _texts(folder)
|
|
for needle in _expected(document):
|
|
assert any(needle in text for text in first.values()), f"{needle!r} not in the derived text"
|
|
|
|
shutil.rmtree(folder)
|
|
assert _texts(_put_and_derive(tree, document)) == first, "the same bytes gave a different text"
|
|
|
|
raw_accept_command(
|
|
files=[Path(f"incoming/{document.stem}")], fidelity="verbatim", authority="reporting",
|
|
page=None, replaces=None, dry_run=False,
|
|
)
|
|
assert not (tree / "incoming" / document.stem).exists()
|