feat(raw): raw derive unpacks ZIP archives and mails with attachments into a folder

A ZIP becomes incoming/<stem>/ with its files and a derived text beside each
document, the ZIP is removed and its sha256 named in the output. A mail with
attachments becomes a folder with original.eml, derived.md and the attachments;
a ZIP attached to a mail is unpacked once, an attached mail is quoted. Unsafe
entries, nested ZIPs and size limits are refused before anything is written.

Files changed:
- CHANGES.md
- VERSION
- docs/how-to/ingest-a-source.md
- docs/reference/commands/README.md
- docs/reference/commands/raw-material-and-uploads.md
- raw/CONTRACT.md
- tools/CONTRACT.md
- tools/README.md
- tools/chemenu/commands/raw_cmd.py
- tools/chemenu/filters/__init__.py
- tools/chemenu/filters/archive.py
- tools/chemenu/filters/mail.py
- tools/chemenu/tests/test_portability.py
- tools/chemenu/tests/test_raw_derive.py

Co-Authored-By: Claude Code <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2

tools/chemenu/tests/test_portability.py

@@ -27,9 +27,10 @@ TOOLS_DIR = Path(__file__).resolve().parents[2]
 PACKAGE_DIR = TOOLS_DIR / "chemenu"
 
 # Receivers whose `open` is not a text-file open: `os.open` takes flags, the
-# archive modules open binary members, and `opener` is a `urllib`
-# `OpenerDirector` (`web_capture.fetch`), whose `open` sends an HTTP request.
-NOT_TEXT_OPEN = {"os", "tarfile", "zipfile", "gzip", "opener"}
+# archive modules open binary members (`archive` is the `zipfile.ZipFile` in
+# `filters/archive.py`), and `opener` is a `urllib` `OpenerDirector`
+# (`web_capture.fetch`), whose `open` sends an HTTP request.
+NOT_TEXT_OPEN = {"os", "tarfile", "zipfile", "gzip", "archive", "opener"}
 
 # The one module allowed to import the platform lock modules.
 LOCK_MODULE = PACKAGE_DIR / "filelock.py"

tools/chemenu/tests/test_raw_derive.py

@@ -5,9 +5,11 @@ readable in the test that uses it and no binary sits in the repository.
 """
 from __future__ import annotations
 
+import hashlib
 import io
 import json
 import zipfile
+from email import message_from_bytes, policy
 from email.message import EmailMessage
 from pathlib import Path
 
@@ -565,14 +567,6 @@ def test_eml_with_inline_image_warns_and_is_not_an_attachment(tree, capsys):
     assert "inline image" in capsys.readouterr().out
 
 
-def test_eml_with_an_attachment_is_refused_untouched(tree, capsys):
-    _put(tree, "mail.eml", make_eml(attach=True))
-    before = _snapshot(tree / "incoming")
-    _refused("incoming/mail.eml")
-    assert "a.bin" in capsys.readouterr().out
-    assert _snapshot(tree / "incoming") == before
-
-
 # --- the registry -----------------------------------------------------------
 
 
@@ -674,3 +668,279 @@ def test_the_web_page_filter_derives_a_saved_html_file(tree):
     assert header["filter"] == "html" and header["title"] == "T"
     assert header["charset"].startswith("utf-8")
     assert "# Kopf" in text
+
+
+# --- containers: ZIP and mail with attachments -------------------------------
+
+
+def make_zip(entries: dict[str, bytes]) -> bytes:
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as zf:
+        for name, content in entries.items():
+            zf.writestr(name, content)
+    return buffer.getvalue()
+
+
+def make_zip_with(info: zipfile.ZipInfo, content: bytes = b"x") -> bytes:
+    encrypted = bool(info.flag_bits & 0x1)
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, "w") as zf:
+        zf.writestr(info, content)
+    data = buffer.getvalue()
+    if encrypted:
+        # zipfile resets the flags on write; set the encryption bit in both headers by hand
+        marker = data.index(b"PK\x01\x02")
+        flags = marker + 8
+        data = data[:flags] + bytes([data[flags] | 1]) + data[flags + 1:]
+    return data
+
+
+def make_mail(attachments: list[tuple[str, bytes, str]], body: str = "See attached.\n") -> bytes:
+    msg = EmailMessage()
+    msg["From"] = "Anna Beispiel <anna@example.org>"
+    msg["To"] = "bob@example.org"
+    msg["Subject"] = "Unterlagen"
+    msg["Date"] = "Wed, 07 Oct 2026 09:30:00 +0200"
+    msg.set_content(body)
+    for name, data, ctype in attachments:
+        maintype, subtype = ctype.split("/")
+        if ctype == "message/rfc822":
+            msg.add_attachment(message_from_bytes(data, policy=policy.default))
+        else:
+            msg.add_attachment(data, maintype=maintype, subtype=subtype, filename=name)
+    return msg.as_bytes()
+
+
+DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+
+
+def _fresh_docx(text: str) -> bytes:
+    return make_docx(_p(text))
+
+
+def test_a_zip_becomes_a_folder_of_its_files_and_is_removed(tree, capsys):
+    one, two = _fresh_docx("One"), _fresh_docx("Two")
+    entries = {"one.docx": one, "sub/two.docx": two, "notes.txt": b"plain\n"}
+    archive = make_zip(entries)
+    _put(tree, "bundle.zip", archive)
+    _derive("incoming/bundle.zip")
+    out = capsys.readouterr().out
+
+    folder = tree / "incoming/bundle"
+    assert (folder / "one.docx").read_bytes() == one
+    assert (folder / "sub/two.docx").read_bytes() == two
+    assert (folder / "notes.txt").read_bytes() == b"plain\n"
+    assert "One" in (folder / "one.md").read_text(encoding="utf-8")
+    assert "Two" in (folder / "sub/two.md").read_text(encoding="utf-8")
+    assert not (tree / "incoming/bundle.zip").exists()
+    assert not (folder / "original.zip").exists() and not (folder / "derived.md").exists()
+    assert "bundle.zip" in out and hashlib.sha256(archive).hexdigest() in out
+    assert "notes.txt" in out
+
+    raw_accept_command(
+        files=[Path("incoming/bundle")], fidelity="verbatim", authority="reporting",
+        page=None, replaces=None, dry_run=False,
+    )
+    assert list((tree / "raw").rglob("one.docx"))
+
+
+def test_a_zip_takes_name_and_names_the_hidden_entries(tree, capsys):
+    _put(tree, "bundle.zip", make_zip({
+        "a.txt": b"a", "__MACOSX/._a.txt": b"x", ".DS_Store": b"x", "sub/.hidden": b"x",
+    }))
+    _derive("incoming/bundle.zip", "kurz")
+    out = capsys.readouterr().out
+    assert sorted(p.name for p in (tree / "incoming/kurz").rglob("*") if p.is_file()) == ["a.txt"]
+    for hidden in ("__MACOSX/._a.txt", ".DS_Store", "sub/.hidden"):
+        assert hidden in out
+
+
+@pytest.mark.parametrize("builder", [
+    lambda: make_zip({"inner.zip": make_zip({"a.txt": b"a"})}),
+    lambda: make_zip({"../escape.txt": b"x"}),
+    lambda: make_zip({"/abs.txt": b"x"}),
+    lambda: make_zip_with(_info("link", symlink=True)),
+    lambda: make_zip_with(_info("secret.txt", encrypted=True)),
+    lambda: b"not a zip at all",
+])
+def test_a_zip_that_breaks_a_rule_is_refused_and_incoming_is_untouched(tree, builder):
+    _put(tree, "bad.zip", builder())
+    before = _snapshot(tree / "incoming")
+    _refused("incoming/bad.zip")
+    assert _snapshot(tree / "incoming") == before
+    assert not (tree / "incoming/bad").exists()
+
+
+def _info(name: str, *, symlink: bool = False, encrypted: bool = False) -> zipfile.ZipInfo:
+    info = zipfile.ZipInfo(name)
+    if symlink:
+        info.external_attr = 0o120777 << 16
+    if encrypted:
+        info.flag_bits |= 0x1
+    return info
+
+
+@pytest.mark.parametrize("limit", ["MAX_ENTRIES", "MAX_FILE_BYTES", "MAX_TOTAL_BYTES"])
+def test_a_zip_over_a_limit_is_refused(tree, monkeypatch, limit):
+    from chemenu.filters import archive
+
+    monkeypatch.setattr(archive, limit, 2 if limit == "MAX_ENTRIES" else 5)
+    _put(tree, "big.zip", make_zip({"a.txt": b"123456", "b.txt": b"123456", "c.txt": b"1"}))
+    before = _snapshot(tree / "incoming")
+    _refused("incoming/big.zip")
+    assert _snapshot(tree / "incoming") == before
+
+
+def test_a_zip_entry_whose_path_would_be_over_budget_is_refused_naming_name(tree, capsys):
+    _put(tree, "z.zip", make_zip({"d" * 100 + "/" + "f" * 80 + ".txt": b"x"}))
+    before = _snapshot(tree / "incoming")
+    _refused("incoming/z.zip")
+    assert "--name" in capsys.readouterr().out
+    assert _snapshot(tree / "incoming") == before
+
+
+def test_a_utf8_entry_name_survives(tree):
+    _put(tree, "u.zip", make_zip({"Übersicht Ä.txt": b"x"}))
+    _derive("incoming/u.zip")
+    assert (tree / "incoming/u/Übersicht Ä.txt").is_file()
+
+
+def test_duplicate_zip_entries_get_a_number(tree, capsys):
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, "w") as zf:
+        zf.writestr("a.txt", b"first")
+        zf.writestr("A.txt", b"second")
+        zf.writestr("_capture.json", b"{}")
+    _put(tree, "d.zip", buffer.getvalue())
+    _derive("incoming/d.zip")
+    out = capsys.readouterr().out
+    names = sorted(p.name for p in (tree / "incoming/d").iterdir())
+    assert names == ["A-2.txt", "_capture-2.json", "a.txt"]
+    assert "A.txt -> A-2.txt" in out
+
+
+def test_a_zip_inside_a_folder_is_left_alone_and_named(tree, capsys):
+    folder = tree / "incoming/f"
+    folder.mkdir()
+    (folder / "a.docx").write_bytes(_fresh_docx("A"))
+    (folder / "z.zip").write_bytes(make_zip({"a.txt": b"a"}))
+    _derive("incoming/f")
+    assert "z.zip" in capsys.readouterr().out
+    assert (folder / "z.zip").is_file() and (folder / "a.md").is_file()
+
+
+def test_a_mail_with_attachments_is_a_folder_of_them(tree, capsys):
+    report = _fresh_docx("Report")
+    inner_one, inner_two = _fresh_docx("Inner one"), _fresh_docx("Inner two")
+    archive = make_zip({"i1.docx": inner_one, "i2.docx": inner_two})
+    raw_mail = make_mail([
+        ("report.docx", report, DOCX),
+        ("papers.zip", archive, "application/zip"),
+        ("picture.bin", b"\x00\x01", "application/octet-stream"),
+    ])
+    _put(tree, "mail.eml", raw_mail)
+    _derive("incoming/mail.eml")
+    out = capsys.readouterr().out
+
+    folder = tree / "incoming/mail"
+    assert (folder / "original.eml").read_bytes() == raw_mail
+    assert (folder / "report.docx").read_bytes() == report
+    assert "Report" in (folder / "report.md").read_text(encoding="utf-8")
+    assert (folder / "papers/i1.docx").read_bytes() == inner_one
+    assert "Inner two" in (folder / "papers/i2.md").read_text(encoding="utf-8")
+    assert (folder / "picture.bin").is_file()
+    assert not (folder / "papers.zip").exists() and not (tree / "incoming/mail.eml").exists()
+    header, text = _header_and_text(folder / "derived.md")
+    assert header["attachments"] == ["report.docx", "papers.zip", "picture.bin"]
+    assert "See attached." in text
+    assert "papers.zip" in out and "sha256" in out
+
+    raw_accept_command(
+        files=[Path("incoming/mail")], fidelity="verbatim", authority="reporting",
+        page=None, replaces=None, dry_run=False,
+    )
+    assert list((tree / "raw").rglob("original.eml"))
+
+
+def test_a_mail_whose_zip_attachment_holds_a_zip_is_refused_untouched(tree):
+    nested = make_zip({"inner.zip": make_zip({"a.txt": b"a"})})
+    _put(tree, "mail.eml", make_mail([("n.zip", nested, "application/zip")]))
+    before = _snapshot(tree / "incoming")
+    _refused("incoming/mail.eml")
+    assert _snapshot(tree / "incoming") == before
+
+
+def test_an_attached_mail_is_quoted_not_unpacked(tree, capsys):
+    inner = make_mail([("deep.bin", b"zz", "application/octet-stream")], body="Inner words\n")
+    _put(tree, "mail.eml", make_mail([("fwd.eml", inner, "message/rfc822")]))
+    _derive("incoming/mail.eml")
+    out = capsys.readouterr().out
+    folder = tree / "incoming/mail"
+    assert sorted(p.name for p in folder.iterdir()) == ["derived.md", "original.eml"]
+    _header, text = _header_and_text(folder / "derived.md")
+    assert "## Attached message: Unterlagen" in text
+    assert "> Inner words" in text
+    assert "deep.bin" in text
+    assert "WARN" in out and "deep.bin" in out
+
+
+def test_a_mail_inside_a_zip_gets_its_md_and_its_attachments_are_only_named(tree, capsys):
+    mail = make_mail([("a.bin", b"x", "application/octet-stream")])
+    _put(tree, "m.zip", make_zip({"msg.eml": mail}))
+    _derive("incoming/m.zip")
+    out = capsys.readouterr().out
+    folder = tree / "incoming/m"
+    assert sorted(p.name for p in folder.iterdir()) == ["msg.eml", "msg.md"]
+    assert "a.bin" in out and "WARN" in out
+
+
+def test_attachment_names_that_collide_get_a_number(tree, capsys):
+    _put(tree, "mail.eml", make_mail([
+        ("a.bin", b"1", "application/octet-stream"),
+        ("a.bin", b"2", "application/octet-stream"),
+        ("derived.md", b"3", "text/markdown"),
+        ("_capture.json", b"4", "application/json"),
+        ("original.docx", _fresh_docx("O"), DOCX),
+        ("../evil/b.bin", b"5", "application/octet-stream"),
+    ]))
+    _derive("incoming/mail.eml")
+    out = capsys.readouterr().out
+    names = sorted(p.name for p in (tree / "incoming/mail").iterdir())
+    assert names == [
+        "_capture-2.json", "a-2.bin", "a.bin", "b.bin", "derived-2.md", "derived.md",
+        "original-2.docx", "original-2.md", "original.eml",
+    ]
+    for renamed in ("a.bin -> a-2.bin", "derived.md -> derived-2.md", "_capture.json -> _capture-2.json"):
+        assert renamed in out
+
+
+def test_a_derived_md_never_takes_the_name_of_a_real_file(tree):
+    _put(tree, "z.zip", make_zip({"a.docx": _fresh_docx("From docx"), "a.md": b"real notes\n"}))
+    _derive("incoming/z.zip")
+    folder = tree / "incoming/z"
+    assert (folder / "a.md").read_bytes() == b"real notes\n"
+    assert "From docx" in (folder / "a-2.md").read_text(encoding="utf-8")
+
+
+def test_a_container_failing_to_write_leaves_incoming_as_it_was(tree, monkeypatch):
+    _put(tree, "bundle.zip", make_zip({"a.txt": b"a", "b.txt": b"b"}))
+    before = _snapshot(tree / "incoming")
+
+    def broken(self):
+        raise OSError("disk full")
+
+    monkeypatch.setattr(Path, "unlink", broken)
+    with pytest.raises(typer.Exit):
+        _derive("incoming/bundle.zip")
+    monkeypatch.undo()
+    assert _snapshot(tree / "incoming") == before
+    assert not (tree / "incoming/bundle").exists()
+
+
+def test_check_lines_for_a_zip_of_xlsx_have_no_mail_line(tree, capsys):
+    _put(tree, "s.zip", make_zip({"sheet.xlsx": make_xlsx()}))
+    _derive("incoming/s.zip")
+    out = capsys.readouterr().out
+    checks = [line for line in out.splitlines() if line.startswith("CHECK")]
+    assert any("Values are as last saved" in line for line in checks)
+    assert not any("commitment" in line for line in checks)