torbenandClaude Code
059a0a1840
Pure restructuring: every raw subcommand's output, help text and exit code is unchanged; the shared helpers sit in raw/_common.py. Files changed: - CHANGES.md - VERSION - tools/README.md - tools/chemenu/cli.py - tools/chemenu/commands/raw/__init__.py - tools/chemenu/commands/raw/_common.py - tools/chemenu/commands/raw/accept.py - tools/chemenu/commands/raw/capture.py - tools/chemenu/commands/raw/derive.py - tools/chemenu/commands/raw/fetch.py - tools/chemenu/commands/raw/pending.py - tools/chemenu/commands/raw_cmd.py - tools/chemenu/tests/test_docs_verify.py - tools/chemenu/tests/test_export_guidelines.py - tools/chemenu/tests/test_raw_capture.py - tools/chemenu/tests/test_raw_cmd.py - tools/chemenu/tests/test_raw_derive.py - tools/chemenu/tests/test_raw_derive_testfiles.py - tools/chemenu/tests/test_raw_fetch.py - tools/chemenu/upload.py Co-Authored-By: Claude Code <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
1213 lines
49 KiB
Python
1213 lines
49 KiB
Python
"""`wikitool raw derive` - reading text from documents that are not text.
|
|
|
|
The documents are built here from `zipfile` and `email`, so every fixture is
|
|
readable in the test that uses it and no binary sits in the repository.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import shutil
|
|
import zipfile
|
|
from email import message_from_bytes, policy
|
|
from email.message import EmailMessage
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import typer
|
|
import yaml
|
|
|
|
from chemenu import filters, web_capture
|
|
from chemenu.commands.raw.accept import raw_accept_command
|
|
from chemenu.commands.raw.derive import raw_derive_command
|
|
from chemenu.commands.raw.pending import raw_pending_command
|
|
|
|
W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
|
|
A = "http://schemas.openxmlformats.org/drawingml/2006/main"
|
|
P = "http://schemas.openxmlformats.org/presentationml/2006/main"
|
|
S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
|
|
R = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"
|
|
REL = "http://schemas.openxmlformats.org/package/2006/relationships"
|
|
DOC_REL = R + "/officeDocument"
|
|
|
|
|
|
def _zip(parts: dict[str, str]) -> bytes:
|
|
buffer = io.BytesIO()
|
|
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
for name, content in parts.items():
|
|
zf.writestr(name, content)
|
|
return buffer.getvalue()
|
|
|
|
|
|
def _rels(target: str, kind: str = DOC_REL, extra: str = "") -> str:
|
|
return (
|
|
f'<Relationships xmlns="{REL}"><Relationship Id="rId1" Type="{kind}" '
|
|
f'Target="{target}"/>{extra}</Relationships>'
|
|
)
|
|
|
|
|
|
def make_docx(body: str, *, comments: str = "", title: str = "") -> bytes:
|
|
parts = {
|
|
"_rels/.rels": _rels("word/document.xml"),
|
|
"word/document.xml": f'<w:document xmlns:w="{W}"><w:body>{body}</w:body></w:document>',
|
|
"word/styles.xml": (
|
|
f'<w:styles xmlns:w="{W}"><w:style w:styleId="Titel"><w:name w:val="heading 1"/></w:style>'
|
|
"</w:styles>"
|
|
),
|
|
"word/numbering.xml": (
|
|
f'<w:numbering xmlns:w="{W}"><w:abstractNum w:abstractNumId="0">'
|
|
'<w:lvl w:ilvl="0"><w:numFmt w:val="decimal"/></w:lvl></w:abstractNum>'
|
|
'<w:abstractNum w:abstractNumId="1"><w:lvl w:ilvl="0"><w:numFmt w:val="bullet"/></w:lvl>'
|
|
'</w:abstractNum><w:num w:numId="1"><w:abstractNumId w:val="0"/></w:num>'
|
|
'<w:num w:numId="2"><w:abstractNumId w:val="1"/></w:num></w:numbering>'
|
|
),
|
|
}
|
|
if comments:
|
|
parts["word/comments.xml"] = f'<w:comments xmlns:w="{W}">{comments}</w:comments>'
|
|
if title:
|
|
parts["docProps/core.xml"] = (
|
|
'<cp:coreProperties xmlns:cp="http://schemas.openxmlformats.org/package/2006/metadata/'
|
|
'core-properties" xmlns:dc="http://purl.org/dc/elements/1.1/">'
|
|
f"<dc:title>{title}</dc:title></cp:coreProperties>"
|
|
)
|
|
return _zip(parts)
|
|
|
|
|
|
def _p(text: str, *, style: str = "", num: tuple[int, int] | None = None) -> str:
|
|
props = ""
|
|
if style:
|
|
props += f'<w:pStyle w:val="{style}"/>'
|
|
if num:
|
|
props += f'<w:numPr><w:ilvl w:val="0"/><w:numId w:val="{num[1]}"/></w:numPr>'
|
|
return f"<w:p><w:pPr>{props}</w:pPr><w:r><w:t>{text}</w:t></w:r></w:p>"
|
|
|
|
|
|
def make_pptx(slides: list[tuple[str, list[str], str]]) -> bytes:
|
|
"""`(title, bullets, notes)` per slide."""
|
|
parts = {"_rels/.rels": _rels("ppt/presentation.xml")}
|
|
ids, rels = "", ""
|
|
for number, (title, bullets, notes) in enumerate(slides, 1):
|
|
ids += f'<p:sldId id="{255 + number}" r:id="rId{number}"/>'
|
|
rels += (
|
|
f'<Relationship Id="rId{number}" Type="{R}/slide" Target="slides/slide{number}.xml"/>'
|
|
)
|
|
body = "".join(f"<a:p><a:r><a:t>{b}</a:t></a:r></a:p>" for b in bullets)
|
|
parts[f"ppt/slides/slide{number}.xml"] = (
|
|
f'<p:sld xmlns:p="{P}" xmlns:a="{A}"><p:cSld><p:spTree>'
|
|
f'<p:sp><p:nvSpPr><p:nvPr><p:ph type="title"/></p:nvPr></p:nvSpPr><p:txBody>'
|
|
f"<a:p><a:r><a:t>{title}</a:t></a:r></a:p></p:txBody></p:sp>"
|
|
f'<p:sp><p:nvSpPr><p:nvPr><p:ph type="body"/></p:nvPr></p:nvSpPr><p:txBody>{body}</p:txBody></p:sp>'
|
|
f'<p:sp><p:nvSpPr><p:nvPr><p:ph type="sldNum"/></p:nvPr></p:nvSpPr><p:txBody>'
|
|
f"<a:p><a:r><a:t>FOOTER-NUMBER</a:t></a:r></a:p></p:txBody></p:sp>"
|
|
"</p:spTree></p:cSld></p:sld>"
|
|
)
|
|
if notes:
|
|
parts[f"ppt/slides/_rels/slide{number}.xml.rels"] = (
|
|
f'<Relationships xmlns="{REL}"><Relationship Id="rId1" Type="{R}/notesSlide" '
|
|
f'Target="../notesSlides/notes{number}.xml"/></Relationships>'
|
|
)
|
|
parts[f"ppt/notesSlides/notes{number}.xml"] = (
|
|
f'<p:notes xmlns:p="{P}" xmlns:a="{A}"><p:cSld><p:spTree><p:sp><p:nvSpPr><p:nvPr>'
|
|
'<p:ph type="body"/></p:nvPr></p:nvSpPr><p:txBody>'
|
|
f"<a:p><a:r><a:t>{notes}</a:t></a:r></a:p></p:txBody></p:sp></p:spTree></p:cSld></p:notes>"
|
|
)
|
|
parts["ppt/presentation.xml"] = (
|
|
f'<p:presentation xmlns:p="{P}" xmlns:r="{R}"><p:sldIdLst>{ids}</p:sldIdLst></p:presentation>'
|
|
)
|
|
parts["ppt/_rels/presentation.xml.rels"] = f'<Relationships xmlns="{REL}">{rels}</Relationships>'
|
|
return _zip(parts)
|
|
|
|
|
|
def make_xlsx() -> bytes:
|
|
sheet_rel = lambda n: f'<Relationship Id="rId{n}" Type="{R}/worksheet" Target="worksheets/sheet{n}.xml"/>' # noqa: E731
|
|
return _zip({
|
|
"_rels/.rels": _rels("xl/workbook.xml"),
|
|
"xl/workbook.xml": (
|
|
f'<workbook xmlns="{S}" xmlns:r="{R}"><sheets>'
|
|
'<sheet name="Budget" sheetId="1" r:id="rId1"/>'
|
|
'<sheet name="Geheim" sheetId="2" state="hidden" r:id="rId2"/></sheets></workbook>'
|
|
),
|
|
"xl/_rels/workbook.xml.rels": (
|
|
f'<Relationships xmlns="{REL}">{sheet_rel(1)}{sheet_rel(2)}'
|
|
f'<Relationship Id="rId3" Type="{R}/sharedStrings" Target="sharedStrings.xml"/>'
|
|
f'<Relationship Id="rId4" Type="{R}/styles" Target="styles.xml"/></Relationships>'
|
|
),
|
|
"xl/sharedStrings.xml": (
|
|
f'<sst xmlns="{S}"><si><t>Posten</t></si><si><t>Miete</t></si>'
|
|
"<si><t>Wert</t></si></sst>"
|
|
),
|
|
"xl/styles.xml": (
|
|
f'<styleSheet xmlns="{S}"><cellXfs><xf numFmtId="0"/><xf numFmtId="14"/></cellXfs></styleSheet>'
|
|
),
|
|
"xl/worksheets/sheet1.xml": (
|
|
f'<worksheet xmlns="{S}"><sheetData>'
|
|
'<row r="2"><c r="B2" t="s"><v>0</v></c><c r="C2" t="s"><v>2</v></c></row>'
|
|
'<row r="3"><c r="B3" t="s"><v>1</v></c><c r="C3"><f>1200+0</f><v>1200</v></c></row>'
|
|
'<row r="4"><c r="B4" t="inlineStr"><is><t>Stichtag</t></is></c><c r="C4" s="1"><v>45292</v></c></row>'
|
|
'<row r="5"><c r="B5" t="inlineStr"><is><t>Fertig</t></is></c><c r="C5" t="b"><v>1</v></c></row>'
|
|
"</sheetData></worksheet>"
|
|
),
|
|
"xl/worksheets/sheet2.xml": (
|
|
f'<worksheet xmlns="{S}"><sheetData><row r="1"><c r="A1" t="inlineStr"><is>'
|
|
"<t>VERSTECKT-INHALT</t></is></c></row></sheetData></worksheet>"
|
|
),
|
|
})
|
|
|
|
|
|
def make_book(sheets: list[tuple[str, str, str]]) -> bytes:
|
|
"""A workbook of `(name, state, <row> elements)` with inline strings only."""
|
|
rels = "".join(
|
|
f'<Relationship Id="rId{n}" Type="{R}/worksheet" Target="worksheets/sheet{n}.xml"/>'
|
|
for n in range(1, len(sheets) + 1)
|
|
)
|
|
listed = "".join(
|
|
f'<sheet name="{name}" sheetId="{n}" state="{state}" r:id="rId{n}"/>'
|
|
for n, (name, state, _rows) in enumerate(sheets, 1)
|
|
)
|
|
parts = {
|
|
"_rels/.rels": _rels("xl/workbook.xml"),
|
|
"xl/workbook.xml": f'<workbook xmlns="{S}" xmlns:r="{R}"><sheets>{listed}</sheets></workbook>',
|
|
"xl/_rels/workbook.xml.rels": f'<Relationships xmlns="{REL}">{rels}</Relationships>',
|
|
}
|
|
for n, (_name, _state, rows) in enumerate(sheets, 1):
|
|
parts[f"xl/worksheets/sheet{n}.xml"] = f'<worksheet xmlns="{S}"><sheetData>{rows}</sheetData></worksheet>'
|
|
return _zip(parts)
|
|
|
|
|
|
def _text_cell(ref: str, text: str) -> str:
|
|
return f'<c r="{ref}" t="inlineStr"><is><t xml:space="preserve">{text.replace(chr(13), " ")}</t></is></c>'
|
|
|
|
|
|
def make_eml(body: str = "Hallo Welt\n", *, html: str = "", alternative: str = "", attach: bool = False,
|
|
inline: bool = False, subject: str = "Angebot") -> bytes:
|
|
msg = EmailMessage()
|
|
msg["From"] = "Anna Beispiel <anna@example.org>"
|
|
msg["To"] = "bob@example.org, Carla <carla@example.org>"
|
|
msg["Subject"] = subject
|
|
msg["Date"] = "Wed, 07 Oct 2026 09:30:00 +0200"
|
|
msg["Message-ID"] = "<abc@example.org>"
|
|
if html:
|
|
msg.set_content(html, subtype="html")
|
|
else:
|
|
msg.set_content(body)
|
|
if alternative:
|
|
msg.add_alternative(alternative, subtype="html")
|
|
if inline:
|
|
msg.add_related(b"\x89PNG", maintype="image", subtype="png", cid="<logo@x>")
|
|
if attach:
|
|
msg.add_attachment(b"data", maintype="application", subtype="octet-stream", filename="a.bin")
|
|
return msg.as_bytes()
|
|
|
|
|
|
@pytest.fixture
|
|
def tree(kb_dir):
|
|
root = kb_dir.parent
|
|
(root / "raw").mkdir()
|
|
(root / "incoming").mkdir(exist_ok=True)
|
|
return root
|
|
|
|
|
|
def _derive(path, name=None):
|
|
return raw_derive_command(path=Path(path), name=name)
|
|
|
|
|
|
def _refused(path, name=None) -> None:
|
|
with pytest.raises(typer.Exit) as excinfo:
|
|
_derive(path, name)
|
|
assert excinfo.value.exit_code == 1
|
|
|
|
|
|
def _snapshot(root: Path) -> dict[str, bytes]:
|
|
return {str(p.relative_to(root)): p.read_bytes() for p in sorted(root.rglob("*")) if p.is_file()}
|
|
|
|
|
|
def _header_and_text(md: Path) -> tuple[dict, str]:
|
|
content = md.read_text(encoding="utf-8")
|
|
assert content.startswith("---\n")
|
|
head, _, text = content[4:].partition("\n---\n")
|
|
return yaml.safe_load(head), text
|
|
|
|
|
|
def _put(tree: Path, name: str, data: bytes) -> Path:
|
|
path = tree / "incoming" / name
|
|
path.write_bytes(data)
|
|
return path
|
|
|
|
|
|
# --- the single-file form ---------------------------------------------------
|
|
|
|
|
|
def test_a_single_file_becomes_a_folder_with_original_and_derived_text(tree):
|
|
data = make_docx(_p("Hello"))
|
|
_put(tree, "Angebot XYZ.docx", data)
|
|
|
|
_derive("incoming/Angebot XYZ.docx")
|
|
|
|
folder = tree / "incoming/Angebot XYZ"
|
|
assert sorted(p.name for p in folder.iterdir()) == ["derived.md", "original.docx"]
|
|
assert (folder / "original.docx").read_bytes() == data
|
|
assert not (tree / "incoming/Angebot XYZ.docx").exists()
|
|
|
|
|
|
def test_the_header_is_valid_yaml_and_records_the_derivation(tree):
|
|
_put(tree, "Angebot XYZ.docx", make_docx(_p("Hello"), title="Offer"))
|
|
_derive("incoming/Angebot XYZ.docx")
|
|
header, text = _header_and_text(tree / "incoming/Angebot XYZ/derived.md")
|
|
|
|
assert header["derived_by"] == "wikitool raw derive"
|
|
assert header["filter"] == "docx"
|
|
assert header["engine"].startswith("wikitool ")
|
|
assert header["derived_from"] == "original.docx"
|
|
assert header["original_name"] == "Angebot XYZ.docx"
|
|
assert header["title"] == "Offer"
|
|
assert header["derived"] is not None
|
|
assert text.strip() == "Hello"
|
|
|
|
|
|
def test_two_derivations_of_the_same_bytes_differ_only_in_stamp_and_file_name(tree):
|
|
data = make_docx(_p("Hello") + _p("World"))
|
|
_put(tree, "a.docx", data)
|
|
_put(tree, "b.docx", data)
|
|
_derive("incoming/a.docx", "same")
|
|
_derive("incoming/b.docx", "same-too")
|
|
|
|
def stripped(name):
|
|
lines = (tree / "incoming" / name / "derived.md").read_text(encoding="utf-8").splitlines()
|
|
return [line for line in lines if not line.startswith(("derived:", "original_name:"))]
|
|
|
|
assert stripped("same") == stripped("same-too")
|
|
|
|
|
|
def test_raw_accept_takes_the_derived_folder_as_one_source(tree):
|
|
data = make_docx(_p("Hello"))
|
|
_put(tree, "report.docx", data)
|
|
_derive("incoming/report.docx")
|
|
|
|
raw_accept_command(
|
|
files=[Path("incoming/report")], fidelity="verbatim", authority="reporting",
|
|
page=None, replaces=None, dry_run=False,
|
|
)
|
|
|
|
[original] = list((tree / "raw").rglob("original.docx"))
|
|
assert original.read_bytes() == data
|
|
assert (original.parent / "derived.md").is_file()
|
|
assert original.parent.name == "report"
|
|
assert not (tree / "incoming/report").exists()
|
|
|
|
|
|
def test_name_picks_the_folder_stem(tree):
|
|
_put(tree, "x.docx", make_docx(_p("Hello")))
|
|
_derive("incoming/x.docx", "kurz")
|
|
assert (tree / "incoming/kurz/original.docx").is_file()
|
|
|
|
|
|
def test_a_120_character_stem_fits_and_an_over_budget_one_is_refused(tree, capsys):
|
|
long_stem = "s" * 120
|
|
_put(tree, f"{long_stem}.docx", make_docx(_p("Hello")))
|
|
_derive(f"incoming/{long_stem}.docx")
|
|
assert (tree / "incoming" / long_stem / "derived.md").is_file()
|
|
capsys.readouterr()
|
|
|
|
too_long = "t" * 170
|
|
_put(tree, f"{too_long}.docx", make_docx(_p("Hello")))
|
|
before = _snapshot(tree / "incoming")
|
|
_refused(f"incoming/{too_long}.docx")
|
|
out = capsys.readouterr().out
|
|
assert "--name" in out and "raw/" in out
|
|
assert _snapshot(tree / "incoming") == before
|
|
_derive(f"incoming/{too_long}.docx", "short")
|
|
assert (tree / "incoming/short/original.docx").is_file()
|
|
|
|
|
|
def test_a_stem_taken_under_raw_is_a_warning_not_a_refusal(tree, capsys):
|
|
(tree / "raw/2026/01").mkdir(parents=True)
|
|
(tree / "raw/2026/01/report.txt").write_text("x", encoding="utf-8")
|
|
_put(tree, "report.docx", make_docx(_p("Hello")))
|
|
_derive("incoming/report.docx")
|
|
out = capsys.readouterr().out
|
|
assert "WARN" in out and "already claims" in out
|
|
assert (tree / "incoming/report/derived.md").is_file()
|
|
|
|
|
|
def test_an_existing_target_folder_is_refused_and_nothing_changes(tree):
|
|
_put(tree, "report.docx", make_docx(_p("Hello")))
|
|
(tree / "incoming/report").mkdir()
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/report.docx")
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
def test_a_file_outside_incoming_or_in_a_subfolder_is_refused(tree):
|
|
outside = tree / "elsewhere.docx"
|
|
outside.write_bytes(make_docx(_p("Hello")))
|
|
_refused(outside)
|
|
(tree / "incoming/sub").mkdir()
|
|
(tree / "incoming/sub/in.docx").write_bytes(make_docx(_p("Hello")))
|
|
_refused("incoming/sub/in.docx")
|
|
assert (tree / "incoming/sub/in.docx").is_file()
|
|
|
|
|
|
def test_a_file_without_a_filter_is_refused_and_names_what_is_read(tree, capsys):
|
|
_put(tree, "notes.md", b"# notes\n")
|
|
_refused("incoming/notes.md")
|
|
out = capsys.readouterr().out
|
|
assert ".docx" in out and ".xlsx" in out
|
|
assert (tree / "incoming/notes.md").is_file()
|
|
|
|
|
|
def test_a_failure_while_moving_the_original_leaves_incoming_as_it_was(tree, monkeypatch):
|
|
_put(tree, "report.docx", make_docx(_p("Hello")))
|
|
before = _snapshot(tree / "incoming")
|
|
|
|
def broken(self, target):
|
|
raise OSError("disk says no")
|
|
|
|
monkeypatch.setattr(Path, "rename", broken)
|
|
_refused("incoming/report.docx")
|
|
monkeypatch.undo()
|
|
assert _snapshot(tree / "incoming") == before
|
|
assert not (tree / "incoming/report").exists()
|
|
|
|
|
|
def test_a_damaged_document_is_refused_untouched(tree):
|
|
_put(tree, "broken.docx", b"this is not a zip")
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/broken.docx")
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
def test_a_lone_markdown_file_stays_folderless(tree):
|
|
_put(tree, "notes.md", b"# notes\n")
|
|
raw_accept_command(
|
|
files=[tree / "incoming/notes.md"], fidelity="verbatim", authority="opinion",
|
|
page=None, replaces=None, dry_run=False,
|
|
)
|
|
[moved] = [p for p in (tree / "raw").rglob("*") if p.is_file()]
|
|
assert moved.name == "notes.md" and moved.parent.name.isdigit()
|
|
|
|
|
|
# --- the folder form --------------------------------------------------------
|
|
|
|
|
|
def test_a_folder_gets_a_md_beside_each_file_with_a_filter(tree, capsys):
|
|
folder = tree / "incoming/mappe"
|
|
folder.mkdir()
|
|
(folder / "plan.docx").write_bytes(make_docx(_p("Plan")))
|
|
(folder / "zahlen.xlsx").write_bytes(make_xlsx())
|
|
(folder / "foto.jpg").write_bytes(b"\xff\xd8")
|
|
_derive("incoming/mappe")
|
|
out = capsys.readouterr().out
|
|
|
|
assert sorted(p.name for p in folder.iterdir()) == ["foto.jpg", "plan.docx", "plan.md", "zahlen.md", "zahlen.sheets", "zahlen.xlsx"]
|
|
assert sorted(p.name for p in (folder / "zahlen.sheets").iterdir()) == [
|
|
"01-budget.formulas.tsv", "01-budget.tsv", "02-geheim.tsv",
|
|
]
|
|
assert "`zahlen.sheets/01-budget.tsv`" in (folder / "zahlen.md").read_text(encoding="utf-8")
|
|
header, _ = _header_and_text(folder / "plan.md")
|
|
assert header["derived_from"] == "plan.docx"
|
|
assert "original_name" not in header
|
|
assert "foto.jpg" in out
|
|
|
|
|
|
def test_a_folder_with_an_existing_md_is_refused_untouched(tree):
|
|
folder = tree / "incoming/mappe"
|
|
folder.mkdir()
|
|
(folder / "plan.docx").write_bytes(make_docx(_p("Plan")))
|
|
(folder / "plan.md").write_text("mine\n", encoding="utf-8")
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/mappe")
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
def test_a_folder_where_two_files_derive_to_one_md_is_refused(tree):
|
|
folder = tree / "incoming/mappe"
|
|
folder.mkdir()
|
|
(folder / "plan.docx").write_bytes(make_docx(_p("Plan")))
|
|
(folder / "plan.xlsx").write_bytes(make_xlsx())
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/mappe")
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
def test_a_folder_without_any_filter_file_is_refused(tree):
|
|
folder = tree / "incoming/mappe"
|
|
folder.mkdir()
|
|
(folder / "foto.jpg").write_bytes(b"\xff\xd8")
|
|
_refused("incoming/mappe")
|
|
|
|
|
|
def test_a_folder_with_a_hidden_entry_is_refused(tree):
|
|
folder = tree / "incoming/mappe"
|
|
folder.mkdir()
|
|
(folder / "plan.docx").write_bytes(make_docx(_p("Plan")))
|
|
(folder / ".DS_Store").write_bytes(b"x")
|
|
_refused("incoming/mappe")
|
|
|
|
|
|
def test_name_does_not_apply_to_a_folder(tree):
|
|
(tree / "incoming/mappe").mkdir()
|
|
(tree / "incoming/mappe/plan.docx").write_bytes(make_docx(_p("Plan")))
|
|
_refused("incoming/mappe", "other")
|
|
|
|
|
|
# --- DOCX -------------------------------------------------------------------
|
|
|
|
|
|
def test_docx_headings_lists_tables_and_text(tree):
|
|
body = (
|
|
_p("Titel", style="Titel")
|
|
+ _p("Absatz eins.")
|
|
+ _p("Erster", num=(0, 1)) + _p("Zweiter", num=(0, 1))
|
|
+ _p("Punkt", num=(0, 2))
|
|
+ f'<w:tbl><w:tr><w:tc>{_p("A")}</w:tc><w:tc>{_p("B")}</w:tc></w:tr>'
|
|
f'<w:tr><w:tc>{_p("1")}</w:tc><w:tc>{_p("2")}</w:tc></w:tr></w:tbl>'
|
|
)
|
|
_put(tree, "d.docx", make_docx(body))
|
|
_derive("incoming/d.docx")
|
|
_, text = _header_and_text(tree / "incoming/d/derived.md")
|
|
|
|
assert "# Titel" in text
|
|
assert "1. Erster\n2. Zweiter" in text
|
|
assert "- Punkt" in text
|
|
assert "| A | B |" in text and "| 1 | 2 |" in text
|
|
|
|
|
|
def test_docx_tracked_changes_and_comments(tree, capsys):
|
|
body = (
|
|
"<w:p><w:r><w:t>Alt </w:t></w:r>"
|
|
'<w:del w:id="1"><w:r><w:delText>GELOESCHT </w:delText></w:r></w:del>'
|
|
'<w:ins w:id="2"><w:r><w:t>NEU</w:t></w:r></w:ins></w:p>'
|
|
)
|
|
comments = '<w:comment w:id="0" w:author="Anna" w:date="2026-10-01"><w:p><w:r><w:t>Prüfen</w:t></w:r></w:p></w:comment>'
|
|
_put(tree, "d.docx", make_docx(body, comments=comments))
|
|
_derive("incoming/d.docx")
|
|
out = capsys.readouterr().out
|
|
header, text = _header_and_text(tree / "incoming/d/derived.md")
|
|
|
|
assert "Alt NEU" in text and "GELOESCHT" not in text
|
|
assert "## Comments" in text and "Anna" in text and "Prüfen" in text
|
|
assert header["tracked_changes"] == 2 and header["comments"] == 1
|
|
assert "CHECK" in out and "Tracked changes" in out and "Comments" in out
|
|
|
|
|
|
def test_docx_without_tracked_changes_prints_no_such_check(tree, capsys):
|
|
_put(tree, "d.docx", make_docx(_p("Plain")))
|
|
_derive("incoming/d.docx")
|
|
out = capsys.readouterr().out
|
|
assert "Tracked changes" not in out and "Comments" not in out
|
|
|
|
|
|
# --- PPTX -------------------------------------------------------------------
|
|
|
|
|
|
def test_pptx_slides_notes_and_footer_placeholders(tree):
|
|
_put(tree, "deck.pptx", make_pptx([("Eins", ["a", "b"], "Sprechnotiz"), ("Zwei", ["c"], "")]))
|
|
_derive("incoming/deck.pptx")
|
|
header, text = _header_and_text(tree / "incoming/deck/derived.md")
|
|
|
|
assert header["slides"] == 2
|
|
assert text.index("## Slide 1") < text.index("## Slide 2")
|
|
assert "**Eins**" in text and "- a" in text and "- b" in text
|
|
assert "### Notes" in text and "Sprechnotiz" in text
|
|
assert "FOOTER-NUMBER" not in text
|
|
|
|
|
|
# --- XLSX -------------------------------------------------------------------
|
|
|
|
|
|
def _tsv_rows(path: Path) -> list[list[str]]:
|
|
with path.open(newline="", encoding="utf-8") as handle:
|
|
return list(csv.reader(handle, delimiter="\t"))
|
|
|
|
|
|
def test_xlsx_becomes_an_overview_and_one_tsv_per_sheet(tree, capsys):
|
|
_put(tree, "zahlen.xlsx", make_xlsx())
|
|
_derive("incoming/zahlen.xlsx")
|
|
out = capsys.readouterr().out
|
|
folder = tree / "incoming/zahlen"
|
|
header, text = _header_and_text(folder / "derived.md")
|
|
|
|
assert header["sheets"] == 2 and header["hidden_sheets"] == ["Geheim"] and header["formulas"] == 1
|
|
assert "## Budget" in text and "## Geheim" in text
|
|
assert "Sheet 2 of 2, hidden" in text
|
|
assert "Range A1:C5 (5 rows x 3 columns), 1 formula cell(s)" in text
|
|
assert "`derived.sheets/01-budget.tsv`" in text and "`derived.sheets/01-budget.formulas.tsv`" in text
|
|
assert "`derived.sheets/02-geheim.tsv`" in text and "02-geheim.formulas" not in text
|
|
assert "Preview" in text and "Posten\tWert" in text
|
|
|
|
# Row n of the file is row n of the sheet, column k its k-th column: the sheet starts at B2.
|
|
assert _tsv_rows(folder / "derived.sheets/01-budget.tsv") == [
|
|
["", "", ""],
|
|
["", "Posten", "Wert"],
|
|
["", "Miete", "1200"],
|
|
["", "Stichtag", "2024-01-01"],
|
|
["", "Fertig", "TRUE"],
|
|
]
|
|
assert _tsv_rows(folder / "derived.sheets/01-budget.formulas.tsv") == [
|
|
["cell", "formula", "value"],
|
|
["C3", "=1200+0", "1200"],
|
|
]
|
|
assert _tsv_rows(folder / "derived.sheets/02-geheim.tsv") == [["VERSTECKT-INHALT"]]
|
|
assert "Values are as last saved" in out and "Hidden sheets" in out and ".formulas.tsv" in out
|
|
assert "derived.sheets" in out
|
|
|
|
|
|
def test_tsv_quotes_per_rfc_4180_and_reads_back_cell_for_cell(tree):
|
|
cells = ["a\tb", 'say "hi"', "two\nlines", "cr\rhere", " padded ", "plain", "="]
|
|
row = "".join(_text_cell(f"{chr(65 + i)}1", c) for i, c in enumerate(cells))
|
|
single = '<row r="1"><c r="A1" t="inlineStr"><is><t></t></is></c></row><row r="2">' + _text_cell("A2", "x") + "</row>"
|
|
_put(tree, "q.xlsx", make_book([("Quoting", "visible", f'<row r="1">{row}</row>'), ("One column", "visible", single)]))
|
|
_derive("incoming/q.xlsx")
|
|
sheets = tree / "incoming/q/derived.sheets"
|
|
|
|
assert _tsv_rows(sheets / "01-quoting.tsv") == [cells]
|
|
raw = (sheets / "01-quoting.tsv").read_bytes()
|
|
assert raw.endswith(b"\n") and b'"a\tb"' in raw and b'"say ""hi"""' in raw and b'"cr\rhere"' in raw
|
|
# A single-column sheet whose first row is empty: the empty row is still a row.
|
|
assert (sheets / "02-one-column.tsv").read_bytes() == b'""\nx\n'
|
|
only_empty_first = make_book([("S", "visible", '<row r="2">' + _text_cell("A2", "x") + "</row>")])
|
|
_put(tree, "e.xlsx", only_empty_first)
|
|
_derive("incoming/e.xlsx")
|
|
assert _tsv_rows(tree / "incoming/e/derived.sheets/01-s.tsv") == [[""], ["x"]]
|
|
assert (tree / "incoming/e/derived.sheets/01-s.tsv").read_bytes() == b'""\nx\n'
|
|
|
|
|
|
def test_shared_array_and_plain_formulas_are_listed_with_their_stored_values(tree, capsys):
|
|
master = "A2*$B$1+B2+SUM(A$1:B$1)+LOG10(A2)&\"A2\"&'My Sheet'!A2"
|
|
rows = (
|
|
'<row r="1"><c r="A1"><v>10</v></c><c r="B1"><v>2</v></c></row>'
|
|
'<row r="2"><c r="A2"><v>20</v></c><c r="B2"><v>3</v></c>'
|
|
f'<c r="C2"><f t="shared" ref="C2:C4" si="0">{master}</f><v>1</v></c></row>'
|
|
'<row r="3"><c r="A3"><v>30</v></c><c r="C3"><f t="shared" si="0"/><v>2</v></c></row>'
|
|
'<row r="4"><c r="C4"><f t="shared" si="0"/><v>3</v></c>'
|
|
'<c r="D4"><f t="array" ref="D4:D5">SUM(A2:A4*B2:B4)</f><v>7</v></c></row>'
|
|
)
|
|
_put(tree, "f.xlsx", make_book([("Calc", "visible", rows)]))
|
|
_derive("incoming/f.xlsx")
|
|
formulas = _tsv_rows(tree / "incoming/f/derived.sheets/01-calc.formulas.tsv")
|
|
|
|
def shifted(n: int) -> str:
|
|
return f"=A{n}*$B$1+B{n}+SUM(A$1:B$1)+LOG10(A{n})&\"A2\"&'My Sheet'!A{n}"
|
|
|
|
assert formulas == [
|
|
["cell", "formula", "value"],
|
|
["C2", shifted(2), "1"],
|
|
["C3", shifted(3), "2"],
|
|
["C4", shifted(4), "3"],
|
|
["D4", "=SUM(A2:A4*B2:B4) (array formula, range D4:D5)", "7"],
|
|
]
|
|
assert "WARN" not in capsys.readouterr().out
|
|
assert _header_and_text(tree / "incoming/f/derived.md")[0]["formulas"] == 4
|
|
|
|
|
|
def test_a_shared_formula_that_would_leave_the_sheet_is_named_by_its_master(tree, capsys):
|
|
rows = (
|
|
'<row r="5"><c r="B5"><f t="shared" ref="B5:B6" si="3">A4+1</f><v>1</v></c></row>'
|
|
'<row r="6"><c r="A6"><f t="shared" si="3"/><v>2</v></c><c r="C6"><f t="shared" si="9"/><v>3</v></c></row>'
|
|
)
|
|
_put(tree, "u.xlsx", make_book([("S", "visible", rows)]))
|
|
_derive("incoming/u.xlsx")
|
|
out = capsys.readouterr().out
|
|
formulas = _tsv_rows(tree / "incoming/u/derived.sheets/01-s.formulas.tsv")
|
|
|
|
assert formulas[1:] == [["B5", "=A4+1", "1"], ["A6", "shared:B5", "2"], ["C6", "shared:si=9", "3"]]
|
|
assert "WARN 2 shared formula cell(s) could not be written out" in out
|
|
|
|
|
|
def test_xlsx_files_are_the_same_on_a_second_run(tree):
|
|
data = make_xlsx()
|
|
_put(tree, "a.xlsx", data)
|
|
_derive("incoming/a.xlsx")
|
|
first = {k: v for k, v in _snapshot(tree / "incoming").items() if k.endswith(".tsv")}
|
|
shutil.rmtree(tree / "incoming/a")
|
|
_put(tree, "a.xlsx", data)
|
|
_derive("incoming/a.xlsx")
|
|
assert {k: v for k, v in _snapshot(tree / "incoming").items() if k.endswith(".tsv")} == first
|
|
assert first
|
|
|
|
|
|
def test_an_empty_and_a_chart_sheet_get_no_file(tree):
|
|
data = make_book([("Leer", "visible", ""), ("Daten", "visible", '<row r="1">' + _text_cell("A1", "x") + "</row>")])
|
|
_put(tree, "l.xlsx", data)
|
|
_derive("incoming/l.xlsx")
|
|
_header, text = _header_and_text(tree / "incoming/l/derived.md")
|
|
assert "*(empty sheet)*" in text
|
|
assert [p.name for p in (tree / "incoming/l/derived.sheets").iterdir()] == ["02-daten.tsv"]
|
|
|
|
|
|
def test_sheet_files_count_against_the_path_budget_and_a_refusal_leaves_incoming_alone(tree, capsys):
|
|
stem = "s" * 120
|
|
_put(tree, f"{stem}.xlsx", make_xlsx())
|
|
before = _snapshot(tree / "incoming")
|
|
_refused(f"incoming/{stem}.xlsx")
|
|
assert "--name" in capsys.readouterr().out
|
|
assert _snapshot(tree / "incoming") == before
|
|
_derive(f"incoming/{stem}.xlsx", "kurz")
|
|
assert (tree / "incoming/kurz/derived.sheets/01-budget.tsv").is_file()
|
|
|
|
|
|
def test_the_folder_form_never_overwrites_a_sheets_folder_and_changes_nothing(tree):
|
|
folder = tree / "incoming/mappe"
|
|
(folder / "zahlen.sheets").mkdir(parents=True)
|
|
(folder / "zahlen.xlsx").write_bytes(make_xlsx())
|
|
before = _snapshot(tree / "incoming")
|
|
(folder / "zahlen.sheets/x.txt").write_text("mine", encoding="utf-8")
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/mappe")
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
def test_a_workbook_in_a_zip_or_a_mail_keeps_its_sheets_beside_its_md(tree):
|
|
_put(tree, "s.zip", make_zip({"sub/Budget 2024.xlsx": make_xlsx(), "Budget 2024.sheets": b"in the way"}))
|
|
_derive("incoming/s.zip")
|
|
folder = tree / "incoming/s"
|
|
assert (folder / "sub/Budget 2024.md").is_file()
|
|
assert sorted(p.name for p in (folder / "sub/Budget 2024.sheets").iterdir()) == [
|
|
"01-budget.formulas.tsv", "01-budget.tsv", "02-geheim.tsv",
|
|
]
|
|
assert "`Budget 2024.sheets/01-budget.tsv`" in (folder / "sub/Budget 2024.md").read_text(encoding="utf-8")
|
|
|
|
_put(tree, "m.eml", make_mail([("derived.xlsx", make_xlsx(), "application/octet-stream")]))
|
|
_derive("incoming/m.eml")
|
|
mail = tree / "incoming/m"
|
|
assert (mail / "derived-2.md").is_file() and (mail / "derived-2.sheets/01-budget.tsv").is_file()
|
|
assert "`derived-2.sheets/01-budget.tsv`" in (mail / "derived-2.md").read_text(encoding="utf-8")
|
|
|
|
|
|
def test_the_derived_folder_with_its_sheet_files_is_accepted_whole(tree):
|
|
_put(tree, "zahlen.xlsx", make_xlsx())
|
|
_derive("incoming/zahlen.xlsx")
|
|
raw_accept_command(
|
|
files=[Path("incoming/zahlen")], fidelity="verbatim", authority="reporting",
|
|
page=None, replaces=None, dry_run=False,
|
|
)
|
|
accepted = sorted(p for p in (tree / "raw").rglob("*.tsv"))
|
|
assert [p.name for p in accepted] == ["01-budget.formulas.tsv", "01-budget.tsv", "02-geheim.tsv"]
|
|
assert not (tree / "incoming/zahlen").exists()
|
|
|
|
|
|
def test_xlsx_check_is_printed_only_for_the_formats_derived_in_the_run(tree, capsys):
|
|
_put(tree, "d.docx", make_docx(_p("Plain")))
|
|
_derive("incoming/d.docx")
|
|
assert "last saved" not in capsys.readouterr().out
|
|
|
|
|
|
def test_check_lines_are_at_most_three_per_format_derived(tree, capsys):
|
|
folder = tree / "incoming/mappe"
|
|
folder.mkdir()
|
|
(folder / "z.xlsx").write_bytes(make_xlsx())
|
|
(folder / "m.eml").write_bytes(make_eml("> quoted\nnew\n"))
|
|
_derive("incoming/mappe")
|
|
out = capsys.readouterr().out
|
|
per_filter = filters.MAX_CHECKS
|
|
assert len([line for line in out.splitlines() if line.startswith("CHECK")]) <= 2 * per_filter
|
|
|
|
|
|
# --- EML --------------------------------------------------------------------
|
|
|
|
|
|
def test_eml_headers_and_plain_body(tree):
|
|
_put(tree, "mail.eml", make_eml("Hallo Welt\n"))
|
|
_derive("incoming/mail.eml")
|
|
header, text = _header_and_text(tree / "incoming/mail/derived.md")
|
|
|
|
assert header["filter"] == "eml"
|
|
assert header["from"] == "Anna Beispiel <anna@example.org>"
|
|
assert header["to"] == ["bob@example.org", "Carla <carla@example.org>"]
|
|
assert header["cc"] == []
|
|
assert header["subject"] == "Angebot" and header["title"] == "Angebot"
|
|
assert header["message_id"] == "<abc@example.org>"
|
|
assert str(header["date"]).startswith("2026-10-07")
|
|
assert header["attachments"] == []
|
|
assert text.strip() == "Hallo Welt"
|
|
|
|
|
|
def test_eml_html_only_is_derived_through_the_web_filter(tree):
|
|
_put(tree, "mail.eml", make_eml(html="<html><body><h1>Gruß</h1><p>Text</p></body></html>"))
|
|
_derive("incoming/mail.eml")
|
|
_, text = _header_and_text(tree / "incoming/mail/derived.md")
|
|
assert "# Gruß" in text and "Text" in text
|
|
|
|
|
|
def test_eml_check_lines_cover_quotes_only_when_they_occur(tree, capsys):
|
|
_put(tree, "plain.eml", make_eml("Fresh text\n"))
|
|
_derive("incoming/plain.eml")
|
|
out = capsys.readouterr().out
|
|
assert "Quoted earlier messages" not in out
|
|
assert "commitment" in out and "secrets" in out
|
|
|
|
_put(tree, "reply.eml", make_eml("Yes.\n\n> earlier\n"))
|
|
_derive("incoming/reply.eml")
|
|
out = capsys.readouterr().out
|
|
assert "Quoted earlier messages" in out
|
|
lines = [line for line in out.splitlines() if line.startswith("CHECK")]
|
|
assert len(lines) == filters.MAX_CHECKS
|
|
|
|
|
|
def test_eml_with_inline_image_warns_and_is_not_an_attachment(tree, capsys):
|
|
_put(tree, "mail.eml", make_eml(html="<p>Hi</p>", inline=True))
|
|
_derive("incoming/mail.eml")
|
|
assert "inline image" in capsys.readouterr().out
|
|
|
|
|
|
_NEWSLETTER = "<html><body><h1>Newsletter</h1><p>" + "Real content of the mail. " * 20 + "</p></body></html>"
|
|
|
|
|
|
def _eml_parsed(data: bytes):
|
|
from chemenu.filters import mail
|
|
|
|
return mail.parse(data)
|
|
|
|
|
|
def test_eml_placeholder_plain_text_gives_way_to_the_html_text(tree, capsys):
|
|
_put(tree, "mail.eml", make_eml("Plain text version not available", alternative=_NEWSLETTER))
|
|
_derive("incoming/mail.eml")
|
|
out = capsys.readouterr().out
|
|
header, text = _header_and_text(tree / "incoming/mail/derived.md")
|
|
|
|
assert "Real content of the mail." in text and "not available" not in text
|
|
assert "The plain text (32 characters) is under 10% of the HTML text" in out
|
|
assert header["attachments"] == []
|
|
assert sorted(p.name for p in (tree / "incoming/mail").iterdir()) == ["derived.md", "original.eml"]
|
|
|
|
|
|
def test_eml_good_plain_text_wins_over_html_without_a_warning(tree, capsys):
|
|
plain = "Real content of the mail. " * 18
|
|
_put(tree, "mail.eml", make_eml(plain, alternative=_NEWSLETTER))
|
|
_derive("incoming/mail.eml")
|
|
out = capsys.readouterr().out
|
|
header, text = _header_and_text(tree / "incoming/mail/derived.md")
|
|
|
|
assert text.strip() == plain.strip()
|
|
assert "is under" not in out
|
|
assert header["attachments"] == []
|
|
|
|
|
|
def test_eml_keeps_a_real_attachment_beside_the_two_text_versions(tree):
|
|
_put(tree, "mail.eml", make_eml("Plain text version not available", alternative=_NEWSLETTER, attach=True))
|
|
_derive("incoming/mail.eml")
|
|
header, _text = _header_and_text(tree / "incoming/mail/derived.md")
|
|
|
|
assert header["attachments"] == ["a.bin"]
|
|
assert (tree / "incoming/mail/a.bin").read_bytes() == b"data"
|
|
|
|
|
|
def test_eml_html_inside_a_related_group_is_still_a_text_version_and_its_image_is_inline():
|
|
msg = EmailMessage()
|
|
msg["Subject"] = "S"
|
|
msg.set_content("Plain text version not available")
|
|
msg.add_alternative(_NEWSLETTER, subtype="html")
|
|
msg.get_payload()[1].add_related(b"\x89PNG", maintype="image", subtype="png", cid="<logo@x>")
|
|
parsed = _eml_parsed(msg.as_bytes())
|
|
|
|
assert "Real content of the mail." in parsed.body
|
|
assert parsed.attachments == []
|
|
assert parsed.inline_images == 1
|
|
|
|
|
|
def test_eml_text_version_marked_as_an_attachment_stays_one():
|
|
msg = EmailMessage()
|
|
msg["Subject"] = "S"
|
|
msg.set_content("Hello there\n")
|
|
msg.add_alternative(_NEWSLETTER, subtype="html")
|
|
msg.get_payload()[1].add_header("Content-Disposition", "attachment", filename="version.html")
|
|
parsed = _eml_parsed(msg.as_bytes())
|
|
|
|
assert parsed.body == "Hello there"
|
|
assert [a.name for a in parsed.attachments] == ["version.html"]
|
|
|
|
|
|
def test_eml_plain_and_html_in_a_mixed_mail_without_alternative_keep_the_html_as_attachment():
|
|
msg = EmailMessage()
|
|
msg["Subject"] = "S"
|
|
msg.set_content("Hello there. " * 40)
|
|
msg.make_mixed()
|
|
html = EmailMessage()
|
|
html.set_content(_NEWSLETTER, subtype="html")
|
|
msg.attach(html)
|
|
parsed = _eml_parsed(msg.as_bytes())
|
|
|
|
assert parsed.body == ("Hello there. " * 40).strip()
|
|
assert len(parsed.attachments) == 1
|
|
|
|
|
|
# --- the registry -----------------------------------------------------------
|
|
|
|
|
|
class _Needy(filters.Filter):
|
|
name = "needy"
|
|
extensions = (".needy",)
|
|
requires = ("no_such_package_for_wikitool",)
|
|
|
|
|
|
def test_a_missing_package_fails_that_format_only(tree, monkeypatch, capsys):
|
|
monkeypatch.setattr(filters, "_REGISTRY", list(filters.all_filters()) + [_Needy()])
|
|
_put(tree, "x.needy", b"data")
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/x.needy")
|
|
out = capsys.readouterr().out
|
|
assert "no_such_package_for_wikitool" in out and "preflight" in out
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
_put(tree, "ok.docx", make_docx(_p("Fine")))
|
|
_derive("incoming/ok.docx")
|
|
assert (tree / "incoming/ok/derived.md").is_file()
|
|
|
|
|
|
def test_registering_a_second_reader_for_an_extension_is_refused():
|
|
class Twin(filters.Filter):
|
|
name = "twin"
|
|
extensions = (".docx",)
|
|
|
|
with pytest.raises(ValueError):
|
|
filters.register(Twin())
|
|
|
|
|
|
def test_a_filter_cannot_ask_for_more_than_three_check_lines():
|
|
with pytest.raises(ValueError):
|
|
filters.Derivation(text="", checks=("a", "b", "c", "d"))
|
|
|
|
|
|
def test_extension_matching_is_case_insensitive_and_ignores_leading_dots():
|
|
assert filters.extension_of("Report.DOCX") == ".docx"
|
|
assert filters.extension_of(".hidden") == ""
|
|
assert filters.filter_for("Report.DOCX").name == "docx"
|
|
assert filters.filter_for("noextension") is None
|
|
|
|
|
|
def test_header_renders_a_list_as_a_flow_sequence_and_a_string_as_before():
|
|
rendered = web_capture.header([("a", "plain"), ("b", ["x y", "z"]), ("c", [])])
|
|
assert rendered == '---\na: plain\nb: ["x y", "z"]\nc: []\n---\n'
|
|
assert yaml.safe_load(rendered.strip("-\n").replace("---", "")) == {"a": "plain", "b": ["x y", "z"], "c": []}
|
|
|
|
|
|
# --- the nets in raw pending and raw accept ---------------------------------
|
|
|
|
|
|
def test_pending_names_the_derive_line_for_a_lone_file_with_a_filter(tree, capsys):
|
|
_put(tree, "report.docx", make_docx(_p("Hello")))
|
|
_put(tree, "photo.jpg", b"\xff")
|
|
raw_pending_command(json_out=True)
|
|
by_path = {c["paths"][0]: c for c in json.loads(capsys.readouterr().out)}
|
|
assert by_path["incoming/report.docx"]["hint"] == "tools/wikitool raw derive incoming/report.docx"
|
|
assert by_path["incoming/photo.jpg"]["hint"] is None
|
|
|
|
raw_pending_command(json_out=False)
|
|
assert "raw derive incoming/report.docx" in capsys.readouterr().out
|
|
|
|
|
|
def test_pending_gives_a_fetch_pair_no_hint(tree, capsys):
|
|
_put(tree, "page.html", b"<p>x</p>")
|
|
_put(tree, "page.md", b"x\n")
|
|
raw_pending_command(json_out=True)
|
|
[candidate] = json.loads(capsys.readouterr().out)
|
|
assert candidate["hint"] is None
|
|
|
|
|
|
def test_accept_warns_about_an_underived_file_but_still_promotes_it(tree, capsys):
|
|
path = _put(tree, "report.docx", make_docx(_p("Hello")))
|
|
raw_accept_command(
|
|
files=[path], fidelity="verbatim", authority="reporting",
|
|
page=None, replaces=None, dry_run=False,
|
|
)
|
|
out = capsys.readouterr().out
|
|
assert "WARN" in out and "raw derive incoming/report.docx" in out
|
|
assert [p for p in (tree / "raw").rglob("report.docx")]
|
|
|
|
|
|
def test_accept_is_quiet_when_a_derived_md_sits_beside_the_file(tree, capsys):
|
|
page = _put(tree, "page.html", b"<p>x</p>")
|
|
md = _put(tree, "page.md", b"x\n")
|
|
raw_accept_command(
|
|
files=[page, md], fidelity="published", authority="reporting",
|
|
page=None, replaces=None, dry_run=False,
|
|
)
|
|
assert "WARN" not in capsys.readouterr().out
|
|
|
|
|
|
def test_the_web_page_filter_derives_a_saved_html_file(tree):
|
|
_put(tree, "seite.html", b"<html><head><title>T</title></head><body><main><h1>Kopf</h1><p>Text</p></main></body></html>")
|
|
_derive("incoming/seite.html")
|
|
header, text = _header_and_text(tree / "incoming/seite/derived.md")
|
|
assert header["filter"] == "html" and header["title"] == "T"
|
|
assert header["charset"].startswith("utf-8")
|
|
assert "# Kopf" in text
|
|
|
|
|
|
# --- containers: ZIP and mail with attachments -------------------------------
|
|
|
|
|
|
def make_zip(entries: dict[str, bytes]) -> bytes:
|
|
buffer = io.BytesIO()
|
|
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
for name, content in entries.items():
|
|
zf.writestr(name, content)
|
|
return buffer.getvalue()
|
|
|
|
|
|
def make_zip_with(info: zipfile.ZipInfo, content: bytes = b"x") -> bytes:
|
|
encrypted = bool(info.flag_bits & 0x1)
|
|
buffer = io.BytesIO()
|
|
with zipfile.ZipFile(buffer, "w") as zf:
|
|
zf.writestr(info, content)
|
|
data = buffer.getvalue()
|
|
if encrypted:
|
|
# zipfile resets the flags on write; set the encryption bit in both headers by hand
|
|
marker = data.index(b"PK\x01\x02")
|
|
flags = marker + 8
|
|
data = data[:flags] + bytes([data[flags] | 1]) + data[flags + 1:]
|
|
return data
|
|
|
|
|
|
def make_mail(attachments: list[tuple[str, bytes, str]], body: str = "See attached.\n") -> bytes:
|
|
msg = EmailMessage()
|
|
msg["From"] = "Anna Beispiel <anna@example.org>"
|
|
msg["To"] = "bob@example.org"
|
|
msg["Subject"] = "Unterlagen"
|
|
msg["Date"] = "Wed, 07 Oct 2026 09:30:00 +0200"
|
|
msg.set_content(body)
|
|
for name, data, ctype in attachments:
|
|
maintype, subtype = ctype.split("/")
|
|
if ctype == "message/rfc822":
|
|
msg.add_attachment(message_from_bytes(data, policy=policy.default))
|
|
else:
|
|
msg.add_attachment(data, maintype=maintype, subtype=subtype, filename=name)
|
|
return msg.as_bytes()
|
|
|
|
|
|
DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
|
|
|
|
|
def _fresh_docx(text: str) -> bytes:
|
|
return make_docx(_p(text))
|
|
|
|
|
|
def test_a_zip_becomes_a_folder_of_its_files_and_is_removed(tree, capsys):
|
|
one, two = _fresh_docx("One"), _fresh_docx("Two")
|
|
entries = {"one.docx": one, "sub/two.docx": two, "notes.txt": b"plain\n"}
|
|
archive = make_zip(entries)
|
|
_put(tree, "bundle.zip", archive)
|
|
_derive("incoming/bundle.zip")
|
|
out = capsys.readouterr().out
|
|
|
|
folder = tree / "incoming/bundle"
|
|
assert (folder / "one.docx").read_bytes() == one
|
|
assert (folder / "sub/two.docx").read_bytes() == two
|
|
assert (folder / "notes.txt").read_bytes() == b"plain\n"
|
|
assert "One" in (folder / "one.md").read_text(encoding="utf-8")
|
|
assert "Two" in (folder / "sub/two.md").read_text(encoding="utf-8")
|
|
assert not (tree / "incoming/bundle.zip").exists()
|
|
assert not (folder / "original.zip").exists() and not (folder / "derived.md").exists()
|
|
assert "bundle.zip" in out and hashlib.sha256(archive).hexdigest() in out
|
|
assert "notes.txt" in out
|
|
|
|
raw_accept_command(
|
|
files=[Path("incoming/bundle")], fidelity="verbatim", authority="reporting",
|
|
page=None, replaces=None, dry_run=False,
|
|
)
|
|
assert list((tree / "raw").rglob("one.docx"))
|
|
|
|
|
|
def test_a_zip_takes_name_and_names_the_hidden_entries(tree, capsys):
|
|
_put(tree, "bundle.zip", make_zip({
|
|
"a.txt": b"a", "__MACOSX/._a.txt": b"x", ".DS_Store": b"x", "sub/.hidden": b"x",
|
|
}))
|
|
_derive("incoming/bundle.zip", "kurz")
|
|
out = capsys.readouterr().out
|
|
assert sorted(p.name for p in (tree / "incoming/kurz").rglob("*") if p.is_file()) == ["a.txt"]
|
|
for hidden in ("__MACOSX/._a.txt", ".DS_Store", "sub/.hidden"):
|
|
assert hidden in out
|
|
|
|
|
|
@pytest.mark.parametrize("builder", [
|
|
lambda: make_zip({"inner.zip": make_zip({"a.txt": b"a"})}),
|
|
lambda: make_zip({"../escape.txt": b"x"}),
|
|
lambda: make_zip({"/abs.txt": b"x"}),
|
|
lambda: make_zip_with(_info("link", symlink=True)),
|
|
lambda: make_zip_with(_info("secret.txt", encrypted=True)),
|
|
lambda: b"not a zip at all",
|
|
])
|
|
def test_a_zip_that_breaks_a_rule_is_refused_and_incoming_is_untouched(tree, builder):
|
|
_put(tree, "bad.zip", builder())
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/bad.zip")
|
|
assert _snapshot(tree / "incoming") == before
|
|
assert not (tree / "incoming/bad").exists()
|
|
|
|
|
|
def _info(name: str, *, symlink: bool = False, encrypted: bool = False) -> zipfile.ZipInfo:
|
|
info = zipfile.ZipInfo(name)
|
|
if symlink:
|
|
info.external_attr = 0o120777 << 16
|
|
if encrypted:
|
|
info.flag_bits |= 0x1
|
|
return info
|
|
|
|
|
|
@pytest.mark.parametrize("limit", ["MAX_ENTRIES", "MAX_FILE_BYTES", "MAX_TOTAL_BYTES"])
|
|
def test_a_zip_over_a_limit_is_refused(tree, monkeypatch, limit):
|
|
from chemenu.filters import archive
|
|
|
|
monkeypatch.setattr(archive, limit, 2 if limit == "MAX_ENTRIES" else 5)
|
|
_put(tree, "big.zip", make_zip({"a.txt": b"123456", "b.txt": b"123456", "c.txt": b"1"}))
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/big.zip")
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
def test_a_zip_entry_whose_path_would_be_over_budget_is_refused_naming_name(tree, capsys):
|
|
_put(tree, "z.zip", make_zip({"d" * 100 + "/" + "f" * 80 + ".txt": b"x"}))
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/z.zip")
|
|
assert "--name" in capsys.readouterr().out
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
def test_a_utf8_entry_name_survives(tree):
|
|
_put(tree, "u.zip", make_zip({"Übersicht Ä.txt": b"x"}))
|
|
_derive("incoming/u.zip")
|
|
assert (tree / "incoming/u/Übersicht Ä.txt").is_file()
|
|
|
|
|
|
def test_duplicate_zip_entries_get_a_number(tree, capsys):
|
|
buffer = io.BytesIO()
|
|
with zipfile.ZipFile(buffer, "w") as zf:
|
|
zf.writestr("a.txt", b"first")
|
|
zf.writestr("A.txt", b"second")
|
|
zf.writestr("_capture.json", b"{}")
|
|
_put(tree, "d.zip", buffer.getvalue())
|
|
_derive("incoming/d.zip")
|
|
out = capsys.readouterr().out
|
|
names = sorted(p.name for p in (tree / "incoming/d").iterdir())
|
|
assert names == ["A-2.txt", "_capture-2.json", "a.txt"]
|
|
assert "A.txt -> A-2.txt" in out
|
|
|
|
|
|
def test_a_zip_inside_a_folder_is_left_alone_and_named(tree, capsys):
|
|
folder = tree / "incoming/f"
|
|
folder.mkdir()
|
|
(folder / "a.docx").write_bytes(_fresh_docx("A"))
|
|
(folder / "z.zip").write_bytes(make_zip({"a.txt": b"a"}))
|
|
_derive("incoming/f")
|
|
assert "z.zip" in capsys.readouterr().out
|
|
assert (folder / "z.zip").is_file() and (folder / "a.md").is_file()
|
|
|
|
|
|
def test_a_mail_with_attachments_is_a_folder_of_them(tree, capsys):
|
|
report = _fresh_docx("Report")
|
|
inner_one, inner_two = _fresh_docx("Inner one"), _fresh_docx("Inner two")
|
|
archive = make_zip({"i1.docx": inner_one, "i2.docx": inner_two})
|
|
raw_mail = make_mail([
|
|
("report.docx", report, DOCX),
|
|
("papers.zip", archive, "application/zip"),
|
|
("picture.bin", b"\x00\x01", "application/octet-stream"),
|
|
])
|
|
_put(tree, "mail.eml", raw_mail)
|
|
_derive("incoming/mail.eml")
|
|
out = capsys.readouterr().out
|
|
|
|
folder = tree / "incoming/mail"
|
|
assert (folder / "original.eml").read_bytes() == raw_mail
|
|
assert (folder / "report.docx").read_bytes() == report
|
|
assert "Report" in (folder / "report.md").read_text(encoding="utf-8")
|
|
assert (folder / "papers/i1.docx").read_bytes() == inner_one
|
|
assert "Inner two" in (folder / "papers/i2.md").read_text(encoding="utf-8")
|
|
assert (folder / "picture.bin").is_file()
|
|
assert not (folder / "papers.zip").exists() and not (tree / "incoming/mail.eml").exists()
|
|
header, text = _header_and_text(folder / "derived.md")
|
|
assert header["attachments"] == ["report.docx", "papers.zip", "picture.bin"]
|
|
assert "See attached." in text
|
|
assert "papers.zip" in out and "sha256" in out
|
|
|
|
raw_accept_command(
|
|
files=[Path("incoming/mail")], fidelity="verbatim", authority="reporting",
|
|
page=None, replaces=None, dry_run=False,
|
|
)
|
|
assert list((tree / "raw").rglob("original.eml"))
|
|
|
|
|
|
def test_a_mail_whose_zip_attachment_holds_a_zip_is_refused_untouched(tree):
|
|
nested = make_zip({"inner.zip": make_zip({"a.txt": b"a"})})
|
|
_put(tree, "mail.eml", make_mail([("n.zip", nested, "application/zip")]))
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/mail.eml")
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
def test_an_attached_mail_is_quoted_not_unpacked(tree, capsys):
|
|
inner = make_mail([("deep.bin", b"zz", "application/octet-stream")], body="Inner words\n")
|
|
_put(tree, "mail.eml", make_mail([("fwd.eml", inner, "message/rfc822")]))
|
|
_derive("incoming/mail.eml")
|
|
out = capsys.readouterr().out
|
|
folder = tree / "incoming/mail"
|
|
assert sorted(p.name for p in folder.iterdir()) == ["derived.md", "original.eml"]
|
|
_header, text = _header_and_text(folder / "derived.md")
|
|
assert "## Attached message: Unterlagen" in text
|
|
assert "> Inner words" in text
|
|
assert "deep.bin" in text
|
|
assert "WARN" in out and "deep.bin" in out
|
|
|
|
|
|
def test_a_mail_inside_a_zip_gets_its_md_and_its_attachments_are_only_named(tree, capsys):
|
|
mail = make_mail([("a.bin", b"x", "application/octet-stream")])
|
|
_put(tree, "m.zip", make_zip({"msg.eml": mail}))
|
|
_derive("incoming/m.zip")
|
|
out = capsys.readouterr().out
|
|
folder = tree / "incoming/m"
|
|
assert sorted(p.name for p in folder.iterdir()) == ["msg.eml", "msg.md"]
|
|
assert "a.bin" in out and "WARN" in out
|
|
|
|
|
|
def test_attachment_names_that_collide_get_a_number(tree, capsys):
|
|
_put(tree, "mail.eml", make_mail([
|
|
("a.bin", b"1", "application/octet-stream"),
|
|
("a.bin", b"2", "application/octet-stream"),
|
|
("derived.md", b"3", "text/markdown"),
|
|
("_capture.json", b"4", "application/json"),
|
|
("original.docx", _fresh_docx("O"), DOCX),
|
|
("../evil/b.bin", b"5", "application/octet-stream"),
|
|
]))
|
|
_derive("incoming/mail.eml")
|
|
out = capsys.readouterr().out
|
|
names = sorted(p.name for p in (tree / "incoming/mail").iterdir())
|
|
assert names == [
|
|
"_capture-2.json", "a-2.bin", "a.bin", "b.bin", "derived-2.md", "derived.md",
|
|
"original-2.docx", "original-2.md", "original.eml",
|
|
]
|
|
for renamed in ("a.bin -> a-2.bin", "derived.md -> derived-2.md", "_capture.json -> _capture-2.json"):
|
|
assert renamed in out
|
|
|
|
|
|
def test_a_derived_md_never_takes_the_name_of_a_real_file(tree):
|
|
_put(tree, "z.zip", make_zip({"a.docx": _fresh_docx("From docx"), "a.md": b"real notes\n"}))
|
|
_derive("incoming/z.zip")
|
|
folder = tree / "incoming/z"
|
|
assert (folder / "a.md").read_bytes() == b"real notes\n"
|
|
assert "From docx" in (folder / "a-2.md").read_text(encoding="utf-8")
|
|
|
|
|
|
def test_a_container_failing_to_write_leaves_incoming_as_it_was(tree, monkeypatch):
|
|
_put(tree, "bundle.zip", make_zip({"a.txt": b"a", "b.txt": b"b"}))
|
|
before = _snapshot(tree / "incoming")
|
|
|
|
def broken(self):
|
|
raise OSError("disk full")
|
|
|
|
monkeypatch.setattr(Path, "unlink", broken)
|
|
with pytest.raises(typer.Exit):
|
|
_derive("incoming/bundle.zip")
|
|
monkeypatch.undo()
|
|
assert _snapshot(tree / "incoming") == before
|
|
assert not (tree / "incoming/bundle").exists()
|
|
|
|
|
|
def test_check_lines_for_a_zip_of_xlsx_have_no_mail_line(tree, capsys):
|
|
_put(tree, "s.zip", make_zip({"sheet.xlsx": make_xlsx()}))
|
|
_derive("incoming/s.zip")
|
|
out = capsys.readouterr().out
|
|
checks = [line for line in out.splitlines() if line.startswith("CHECK")]
|
|
assert any("Values are as last saved" in line for line in checks)
|
|
assert not any("commitment" in line for line in checks)
|