chemenu/tools/chemenu/tests/test_raw_derive_binary.py

"""`wikitool raw derive` for the two formats read through a package: PDF (`pypdf`) and
Outlook `.msg` (`olefile`). The documents come from `binary_formats`, so no binary sits
in the repository; whether the builders write valid files is settled by the packages
reading them here. The tests that need a package skip where it is not installed - CI
installs `tools/requirements.txt`, so they run there."""
from __future__ import annotations

import importlib.util
import io
from pathlib import Path

import pytest

from chemenu import filters
from chemenu.commands import doctor
from chemenu.tests.binary_formats import build_msg, build_pdf
from chemenu.tests.test_raw_derive import (  # noqa: F401  (tree is a fixture)
    _derive,
    _header_and_text,
    _put,
    _refused,
    _snapshot,
    make_docx,
    make_mail,
    make_zip,
    tree,
)

needs_pypdf = pytest.mark.skipif(importlib.util.find_spec("pypdf") is None, reason="pypdf is not installed")
needs_olefile = pytest.mark.skipif(importlib.util.find_spec("olefile") is None, reason="olefile is not installed")

LONG_LINE = "A sentence long enough that the page is clearly not a scan of a picture."


# --- PDF ---------------------------------------------------------------------


@needs_pypdf
def test_a_pdf_becomes_a_folder_with_page_markers(tree, capsys):
    original = build_pdf([[LONG_LINE, "Second line (with brackets)"], [LONG_LINE + " Two."]], title="Quarterly report")
    _put(tree, "Report.pdf", original)
    _derive("incoming/Report.pdf")
    out = capsys.readouterr().out

    folder = tree / "incoming/Report"
    assert (folder / "original.pdf").read_bytes() == original
    header, text = _header_and_text(folder / "derived.md")
    assert header["filter"] == "pdf" and "pypdf" in header["engine"]
    assert int(header["pages"]) == 2 and header["title"] == "Quarterly report"
    assert text.index("<!-- page 1 -->") < text.index("Second line (with brackets)") < text.index("<!-- page 2 -->")
    assert "WARN" not in out
    assert out.count("CHECK") == 1 and "page" in out


@needs_pypdf
def test_a_pdf_without_text_is_derived_with_a_scan_warning(tree, capsys):
    _put(tree, "scan.pdf", build_pdf([[], [], []]))
    _derive("incoming/scan.pdf")
    out = capsys.readouterr().out
    assert "WARN" in out and "scan" in out and "3 page" in out
    assert (tree / "incoming/scan/original.pdf").is_file()


@needs_pypdf
def test_a_pdf_that_is_not_a_pdf_is_refused_untouched(tree):
    _put(tree, "fake.pdf", b"this is not a pdf at all")
    before = _snapshot(tree / "incoming")
    _refused("incoming/fake.pdf")
    assert _snapshot(tree / "incoming") == before


@needs_pypdf
def test_an_encrypted_pdf_is_refused_untouched(tree, capsys):
    import pypdf

    writer = pypdf.PdfWriter(clone_from=pypdf.PdfReader(io.BytesIO(build_pdf([[LONG_LINE]]))))
    try:
        writer.encrypt("secret", algorithm="RC4-128")
    except Exception as exc:  # no crypto provider for the algorithm on this machine
        pytest.skip(f"cannot build an encrypted PDF here: {exc}")
    buffer = io.BytesIO()
    writer.write(buffer)
    _put(tree, "locked.pdf", buffer.getvalue())
    before = _snapshot(tree / "incoming")
    _refused("incoming/locked.pdf")
    assert "encrypted" in capsys.readouterr().out
    assert _snapshot(tree / "incoming") == before


@needs_pypdf
def test_a_pdf_inside_a_zip_gets_its_md(tree):
    _put(tree, "bundle.zip", make_zip({"a.pdf": build_pdf([[LONG_LINE]]), "b.txt": b"x"}))
    _derive("incoming/bundle.zip")
    folder = tree / "incoming/bundle"
    assert LONG_LINE in (folder / "a.md").read_text(encoding="utf-8")
    assert (folder / "a.pdf").is_file() and not (folder / "b.md").exists()


@needs_pypdf
def test_a_pdf_attached_to_a_mail_gets_its_md(tree):
    _put(tree, "mail.eml", make_mail([("offer.pdf", build_pdf([[LONG_LINE]]), "application/pdf")]))
    _derive("incoming/mail.eml")
    assert LONG_LINE in (tree / "incoming/mail/offer.md").read_text(encoding="utf-8")


def test_a_missing_package_stops_only_its_own_format(tree, monkeypatch, capsys):
    pdf_filter = filters.filter_for("a.pdf")
    monkeypatch.setattr(type(pdf_filter), "missing", lambda self: ["pypdf"])
    _put(tree, "a.pdf", b"%PDF-1.4")
    _put(tree, "b.docx", make_docx("<w:p><w:r><w:t>Still works</w:t></w:r></w:p>"))
    before = _snapshot(tree / "incoming")
    _refused("incoming/a.pdf")
    out = capsys.readouterr().out
    assert "pypdf" in out and "preflight" in out
    assert _snapshot(tree / "incoming") == before
    _derive("incoming/b.docx")
    assert "Still works" in (tree / "incoming/b/derived.md").read_text(encoding="utf-8")


def test_a_package_missing_inside_a_zip_refuses_the_whole_zip(tree, monkeypatch):
    monkeypatch.setattr(type(filters.filter_for("a.pdf")), "missing", lambda self: ["pypdf"])
    _put(tree, "bundle.zip", make_zip({"a.pdf": b"%PDF-1.4", "b.txt": b"x"}))
    before = _snapshot(tree / "incoming")
    _refused("incoming/bundle.zip")
    assert _snapshot(tree / "incoming") == before


# --- Outlook .msg --------------------------------------------------------------


SENT = 134_000_000_000_000_000  # 2025-08-18T14:13:20+00:00


@needs_olefile
def test_a_msg_becomes_a_folder_like_a_mail(tree, capsys):
    data = [b"attachment bytes" * 400, b"small"]
    original = build_msg(
        subject="Unterlagen", body="Hello Bob\r\nSee attached.", sender="Anna", sender_address="anna@example.org",
        to="bob@example.org; carol@example.org", sent=SENT,
        attachments=[("report.bin", data[0]), ("note.txt", data[1])],
    )
    _put(tree, "Mail.msg", original)
    _derive("incoming/Mail.msg")
    out = capsys.readouterr().out

    folder = tree / "incoming/Mail"
    assert (folder / "original.msg").read_bytes() == original
    assert (folder / "report.bin").read_bytes() == data[0]
    assert (folder / "note.txt").read_bytes() == data[1]
    header, text = _header_and_text(folder / "derived.md")
    assert header["filter"] == "msg" and "olefile" in header["engine"]
    assert header["from"] == "Anna <anna@example.org>"
    assert header["to"] == ["bob@example.org", "carol@example.org"]
    assert header["subject"] == "Unterlagen" and str(header["date"]).startswith("2025-08-18")
    assert header["attachments"] == ["report.bin", "note.txt"]
    assert text.strip() == "Hello Bob\nSee attached."
    assert "report.bin" in out


@needs_olefile
def test_a_msg_with_only_html_takes_the_html_text(tree):
    _put(tree, "h.msg", build_msg(subject="S", html="<html><body><p>Only <b>html</b> here</p></body></html>"))
    _derive("incoming/h.msg")
    _header, text = _header_and_text(tree / "incoming/h/derived.md")
    assert "Only" in text and "html here" in text and "<p>" not in text


_HTML_BODY = "<html><body><p>" + "Real content of the mail. " * 20 + "</p></body></html>"


@needs_olefile
def test_a_msg_with_a_placeholder_plain_text_takes_the_html_text_and_warns(tree, capsys):
    _put(tree, "p.msg", build_msg(subject="S", body="Plain text version not available", html=_HTML_BODY))
    _derive("incoming/p.msg")
    out = capsys.readouterr().out
    _header, text = _header_and_text(tree / "incoming/p/derived.md")

    assert "Real content of the mail." in text and "not available" not in text
    assert "The plain text (32 characters) is under 10% of the HTML text" in out


@needs_olefile
def test_a_msg_with_a_good_plain_text_keeps_it_without_a_warning(tree, capsys):
    plain = "Real content of the mail. " * 18
    _put(tree, "g.msg", build_msg(subject="S", body=plain, html=_HTML_BODY))
    _derive("incoming/g.msg")
    out = capsys.readouterr().out
    _header, text = _header_and_text(tree / "incoming/g/derived.md")

    assert text.strip() == plain.strip()
    assert "is under" not in out


@needs_olefile
def test_a_msg_with_only_rtf_warns_and_has_no_text(tree, capsys):
    _put(tree, "r.msg", build_msg(subject="S", rtf=b"{\\rtf1 hello}"))
    _derive("incoming/r.msg")
    out = capsys.readouterr().out
    assert "WARN" in out and "RTF" in out
    _header, text = _header_and_text(tree / "incoming/r/derived.md")
    assert text.strip() == ""


@needs_olefile
def test_a_msg_names_inline_images_and_embedded_messages_without_writing_them(tree, capsys):
    _put(tree, "m.msg", build_msg(
        subject="S", body="text", inline_images=[("logo.png", b"png")], embedded=["forwarded.msg"],
        attachments=[("real.bin", b"x")],
    ))
    _derive("incoming/m.msg")
    out = capsys.readouterr().out
    assert sorted(p.name for p in (tree / "incoming/m").iterdir()) == ["derived.md", "original.msg", "real.bin"]
    assert "inline image" in out and "forwarded.msg" in out


@needs_olefile
def test_a_msg_attached_to_a_mail_is_quoted(tree, capsys):
    inner = build_msg(subject="Inner subject", body="Inner words", sender="Dora", to="x@example.org")
    _put(tree, "mail.eml", make_mail([("inner.msg", inner, "application/vnd.ms-outlook")]))
    _derive("incoming/mail.eml")
    folder = tree / "incoming/mail"
    assert sorted(p.name for p in folder.iterdir()) == ["derived.md", "original.eml"]
    _header, text = _header_and_text(folder / "derived.md")
    assert "## Attached message: Inner subject" in text and "> Inner words" in text


@needs_olefile
def test_a_file_that_is_not_a_msg_is_refused_untouched(tree):
    _put(tree, "x.msg", b"plain text pretending to be a message")
    before = _snapshot(tree / "incoming")
    _refused("incoming/x.msg")
    assert _snapshot(tree / "incoming") == before


@needs_olefile
def test_a_compound_file_that_is_not_a_message_is_refused(tree):
    from chemenu.tests.binary_formats import build_cfb

    _put(tree, "x.msg", build_cfb({"WordDocument": b"abc"}))
    _refused("incoming/x.msg")
    assert not (tree / "incoming/x").exists()


def test_a_msg_without_the_package_is_refused_and_a_docx_still_works(tree, monkeypatch, capsys):
    monkeypatch.setattr(type(filters.filter_for("a.msg")), "missing", lambda self: ["olefile"])
    _put(tree, "a.msg", b"\xd0\xcf\x11\xe0")
    before = _snapshot(tree / "incoming")
    _refused("incoming/a.msg")
    assert "olefile" in capsys.readouterr().out
    assert _snapshot(tree / "incoming") == before


# --- doctor ---------------------------------------------------------------------


def test_doctor_names_each_filter_that_needs_a_package():
    names = {check.name for check in doctor.check_import_filters()}
    assert names == {"import-filter-pdf", "import-filter-msg"}


def test_doctor_warns_for_a_missing_package_and_never_fails(monkeypatch):
    monkeypatch.setattr(type(filters.filter_for("a.pdf")), "missing", lambda self: ["pypdf"])
    checks = {check.name: check for check in doctor.check_import_filters()}
    assert checks["import-filter-pdf"].status == "WARN"
    assert "pypdf" in checks["import-filter-pdf"].detail and checks["import-filter-pdf"].fix
    assert all(check.status != "FAIL" for check in checks.values())


@needs_pypdf
def test_doctor_reports_an_installed_package_with_its_version():
    check = next(c for c in doctor.check_import_filters() if c.name == "import-filter-pdf")
    assert check.status == "OK" and check.detail.startswith("pypdf ") and ".pdf" in check.detail