torbenandClaude Code
128c7f5c59
Files changed: - CHANGES.md - VERSION - docs/how-to/ingest-a-source.md - docs/reference/commands/raw-material-and-uploads.md - instructions/ingest-large-tree.md - instructions/read-page-images.md - instructions/wiki-ingest/SKILL.md - raw/CONTRACT.md - tools/CONTRACT.md - tools/chemenu/commands/raw/derive.py - tools/chemenu/filters/pdf.py - tools/chemenu/filters/pdf_layout.py - tools/chemenu/tests/test_raw_derive_binary.py - types/source.guidance.md Co-Authored-By: Claude Code <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
666 lines
29 KiB
Python
666 lines
29 KiB
Python
"""`wikitool raw derive` for the two formats read through a package: PDF (`pypdf` for the text and
|
|
the outline, `pdfminer.six` for the look of the characters) and Outlook `.msg` (`olefile`). The documents come from `binary_formats`, so no binary sits
|
|
in the repository; whether the builders write valid files is settled by the packages
|
|
reading them here. The tests that need a package skip where it is not installed - CI
|
|
installs `tools/requirements.txt`, so they run there."""
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import importlib.util
|
|
import io
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from chemenu import filters
|
|
from chemenu.commands import doctor
|
|
from chemenu.filters import pdf_layout
|
|
from chemenu.tests.binary_formats import Picture, StyledPage, build_msg, build_pdf, build_styled_pdf
|
|
from chemenu.tests.test_raw_derive import ( # noqa: F401 (tree is a fixture)
|
|
_derive,
|
|
_header_and_text,
|
|
_put,
|
|
_refused,
|
|
_snapshot,
|
|
make_docx,
|
|
make_mail,
|
|
make_zip,
|
|
tree,
|
|
)
|
|
|
|
needs_pypdf = pytest.mark.skipif(importlib.util.find_spec("pypdf") is None, reason="pypdf is not installed")
|
|
needs_pdfminer = pytest.mark.skipif(importlib.util.find_spec("pdfminer") is None, reason="pdfminer.six is not installed")
|
|
needs_olefile = pytest.mark.skipif(importlib.util.find_spec("olefile") is None, reason="olefile is not installed")
|
|
|
|
LONG_LINE = "A sentence long enough that the page is clearly not a scan of a picture."
|
|
|
|
|
|
# --- PDF ---------------------------------------------------------------------
|
|
|
|
|
|
@needs_pypdf
|
|
def test_a_pdf_becomes_a_folder_with_page_markers(tree, capsys):
|
|
original = build_pdf([[LONG_LINE, "Second line (with brackets)"], [LONG_LINE + " Two."]], title="Quarterly report")
|
|
_put(tree, "Report.pdf", original)
|
|
_derive("incoming/Report.pdf")
|
|
out = capsys.readouterr().out
|
|
|
|
folder = tree / "incoming/Report"
|
|
assert (folder / "original.pdf").read_bytes() == original
|
|
header, text = _header_and_text(folder / "derived.md")
|
|
assert header["filter"] == "pdf" and "pypdf" in header["engine"]
|
|
assert int(header["pages"]) == 2 and header["title"] == "Quarterly report"
|
|
assert text.index("<!-- page 1 -->") < text.index("Second line (with brackets)") < text.index("<!-- page 2 -->")
|
|
assert "WARN" not in out
|
|
assert out.count("CHECK") == 1 and "page" in out
|
|
|
|
|
|
@needs_pypdf
|
|
def test_a_pdf_without_text_is_derived_with_a_scan_warning(tree, capsys):
|
|
_put(tree, "scan.pdf", build_pdf([[], [], []]))
|
|
_derive("incoming/scan.pdf")
|
|
out = capsys.readouterr().out
|
|
assert "WARN" in out and "scan" in out and "3 page" in out
|
|
assert (tree / "incoming/scan/original.pdf").is_file()
|
|
|
|
|
|
@needs_pypdf
|
|
def test_a_pdf_that_is_not_a_pdf_is_refused_untouched(tree):
|
|
_put(tree, "fake.pdf", b"this is not a pdf at all")
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/fake.pdf")
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
@needs_pypdf
|
|
def test_an_encrypted_pdf_is_refused_untouched(tree, capsys):
|
|
import pypdf
|
|
|
|
writer = pypdf.PdfWriter(clone_from=pypdf.PdfReader(io.BytesIO(build_pdf([[LONG_LINE]]))))
|
|
try:
|
|
writer.encrypt("secret", algorithm="RC4-128")
|
|
except Exception as exc: # no crypto provider for the algorithm on this machine
|
|
pytest.skip(f"cannot build an encrypted PDF here: {exc}")
|
|
buffer = io.BytesIO()
|
|
writer.write(buffer)
|
|
_put(tree, "locked.pdf", buffer.getvalue())
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/locked.pdf")
|
|
assert "encrypted" in capsys.readouterr().out
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
@needs_pypdf
|
|
def test_a_pdf_inside_a_zip_gets_its_md(tree):
|
|
_put(tree, "bundle.zip", make_zip({"a.pdf": build_pdf([[LONG_LINE]]), "b.txt": b"x"}))
|
|
_derive("incoming/bundle.zip")
|
|
folder = tree / "incoming/bundle"
|
|
assert LONG_LINE in (folder / "a.md").read_text(encoding="utf-8")
|
|
assert (folder / "a.pdf").is_file() and not (folder / "b.md").exists()
|
|
|
|
|
|
@needs_pypdf
|
|
def test_a_pdf_attached_to_a_mail_gets_its_md(tree):
|
|
_put(tree, "mail.eml", make_mail([("offer.pdf", build_pdf([[LONG_LINE]]), "application/pdf")]))
|
|
_derive("incoming/mail.eml")
|
|
assert LONG_LINE in (tree / "incoming/mail/offer.md").read_text(encoding="utf-8")
|
|
|
|
|
|
def test_a_missing_package_stops_only_its_own_format(tree, monkeypatch, capsys):
|
|
pdf_filter = filters.filter_for("a.pdf")
|
|
monkeypatch.setattr(type(pdf_filter), "missing", lambda self: ["pypdf"])
|
|
_put(tree, "a.pdf", b"%PDF-1.4")
|
|
_put(tree, "b.docx", make_docx("<w:p><w:r><w:t>Still works</w:t></w:r></w:p>"))
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/a.pdf")
|
|
out = capsys.readouterr().out
|
|
assert "pypdf" in out and "preflight" in out
|
|
assert _snapshot(tree / "incoming") == before
|
|
_derive("incoming/b.docx")
|
|
assert "Still works" in (tree / "incoming/b/derived.md").read_text(encoding="utf-8")
|
|
|
|
|
|
def test_a_package_missing_inside_a_zip_refuses_the_whole_zip(tree, monkeypatch):
|
|
monkeypatch.setattr(type(filters.filter_for("a.pdf")), "missing", lambda self: ["pypdf"])
|
|
_put(tree, "bundle.zip", make_zip({"a.pdf": b"%PDF-1.4", "b.txt": b"x"}))
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/bundle.zip")
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
# --- PDF: outline, markings, images --------------------------------------------
|
|
|
|
YELLOW, GREEN, RED, BLUE = (1, 1, 0), (0, 1, 0), (1, 0, 0), (0, 0, 1)
|
|
|
|
|
|
def _derive_pdf(tree, pdf: bytes, name: str = "doc.pdf"):
|
|
_put(tree, name, pdf)
|
|
_derive(f"incoming/{name}")
|
|
return _header_and_text(tree / "incoming" / name[:-4] / "derived.md")
|
|
|
|
|
|
def _markings_stripped(text: str) -> str:
|
|
return re.sub(r"~~|\*", "", text)
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_the_outline_goes_into_the_header_with_level_page_and_line(tree):
|
|
pdf = build_styled_pdf(
|
|
[StyledPage([["One"]]), StyledPage([["Two", ], ["more"]]), StyledPage([["Three"]])],
|
|
outline=[("Part A", 1, 0), ("Rule 16 (hands)", 2, 1), ("Signals", 3, 1), ("Part B", 1, 2), ("Lost entry", 2, None)],
|
|
)
|
|
_put(tree, "doc.pdf", pdf)
|
|
_derive("incoming/doc.pdf")
|
|
path = tree / "incoming/doc/derived.md"
|
|
header, _text = _header_and_text(path)
|
|
file_lines = path.read_text(encoding="utf-8").split("\n")
|
|
|
|
assert header["outline"] == [
|
|
"Part A | level 1 | page 1 | line " + str(file_lines.index("<!-- page 1 -->") + 1),
|
|
"Rule 16 (hands) | level 2 | page 2 | line " + str(file_lines.index("<!-- page 2 -->") + 1),
|
|
"Signals | level 3 | page 2 | line " + str(file_lines.index("<!-- page 2 -->") + 1),
|
|
"Part B | level 1 | page 3 | line " + str(file_lines.index("<!-- page 3 -->") + 1),
|
|
"Lost entry | level 2",
|
|
]
|
|
for entry in header["outline"][:4]:
|
|
_title, _level, page, line = entry.rsplit(" | ", 3)
|
|
assert file_lines[int(line.split()[1]) - 1] == f"<!-- {page} -->"
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_a_pdf_without_outline_or_markings_has_neither_field_and_no_image_check(tree, capsys):
|
|
header, text = _derive_pdf(tree, build_styled_pdf([StyledPage([[LONG_LINE]])], title="T"))
|
|
out = capsys.readouterr().out
|
|
assert "outline" not in header and "markings" not in header
|
|
assert "images" not in header and "unmatched" not in header
|
|
assert "*" not in text and "WARN" not in out and out.count("CHECK") == 1
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_a_highlight_behind_text_becomes_a_strong_mark_and_the_legend_names_the_colours(tree):
|
|
pdf = build_styled_pdf([
|
|
StyledPage([["Intro ", ("old rule", {"highlight": YELLOW}), " and ", ("new rule", {"highlight": GREEN}), " end"]]),
|
|
StyledPage([[("second yellow", {"highlight": YELLOW})]]),
|
|
])
|
|
header, text = _derive_pdf(tree, pdf)
|
|
assert "Intro **old rule** and **new rule** end" in text and "**second yellow**" in text
|
|
(entry,) = [e for e in header["markings"] if e.startswith("**…**")]
|
|
assert "places 3 | pages 1-2" in entry
|
|
assert "gelb on page(s) 1-2" in entry and "grün on page(s) 1" in entry
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_only_a_bright_saturated_fill_about_as_wide_as_its_text_is_a_highlight(tree):
|
|
dark = "0.2 0.3 0.5 rg 72 700 200 14 re f 0 g" # no highlighter is dark
|
|
panel = "1 1 0 rg 60 650 480 40 re f 0 g" # a box far bigger than the line it holds
|
|
wide = "1 1 0 rg 60 612 480 12 re f 0 g" # as high as a line, but wider than the words
|
|
grey = "0.85 g 72 580 200 12 re f 0 g" # shading, not a pen
|
|
pdf = build_styled_pdf([StyledPage(
|
|
[[("on white", {"highlight": (1, 1, 1)})], ["on dark"], ["in a panel"], ["short"], ["on grey"]], ops=[dark, panel, wide, grey],
|
|
)])
|
|
header, text = _derive_pdf(tree, pdf)
|
|
assert "*" not in text and "markings" not in header
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_bold_and_a_different_colour_are_strong_but_only_inside_running_text(tree):
|
|
pdf = build_styled_pdf([StyledPage([
|
|
["Running text in the body colour, ", ("alert", {"color": RED}), " and ", ("see page 3", {"color": BLUE, "link": True}), " and a ", ("bold", {"bold": True}), " word."],
|
|
[("A whole line of heading text in bold", {"bold": True})],
|
|
["Label ", ("bold", {"bold": True})],
|
|
[("A line that is red from end to end, a banner", {"color": RED})],
|
|
])])
|
|
header, text = _derive_pdf(tree, pdf)
|
|
assert "colour, **alert** and see page 3 and a **bold** word." in text
|
|
assert "\nA whole line of heading text in bold\n" in text
|
|
assert "\nLabel bold\n" in text and "\nA line that is red from end to end, a banner\n" in text
|
|
(entry,) = [e for e in header["markings"] if e.startswith("**…**")]
|
|
assert "places 2" in entry and "bold" in entry and "colour" in entry
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_the_running_text_colour_is_the_one_with_most_characters_not_black(tree):
|
|
navy, gray_level = (0, 0.18, 0.49), 0.4
|
|
pdf = build_styled_pdf([StyledPage([[
|
|
("Body text that is set in navy, ", {"color": navy}), ("margin", {"color": (0.9, 0.2, 0.2)}), (" more navy body text.", {"color": navy}),
|
|
]])])
|
|
_header, text = _derive_pdf(tree, pdf)
|
|
assert "Body text that is set in navy, **margin** more navy body text." in text
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_italic_is_marked_inside_running_text_and_not_when_the_whole_line_is_italic(tree):
|
|
pdf = build_styled_pdf([StyledPage([
|
|
["Running text with an ", ("emphasised", {"italic": True}), " word in the middle."],
|
|
[("A whole line of quotation in italic type", {"italic": True})],
|
|
])])
|
|
header, text = _derive_pdf(tree, pdf)
|
|
assert "an *emphasised* word" in text and "\nA whole line of quotation in italic type\n" in text
|
|
assert [e for e in header["markings"] if e.startswith("*…*")]
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_a_line_through_text_becomes_a_strike_through(tree):
|
|
pdf = build_styled_pdf([StyledPage([["Keep ", ("dropped", {"strike": True}), " keep"]])])
|
|
header, text = _derive_pdf(tree, pdf)
|
|
assert "Keep ~~dropped~~ keep" in text
|
|
assert any(entry.startswith("~~…~~") and "places 1" in entry for entry in header["markings"])
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_a_line_that_is_not_the_characters_own_is_no_strike_through(tree):
|
|
rule = "0 G 0.7 w 72 723 m 540 723 l S" # through the middle of "short", but running far past it
|
|
one = "0 G 0.7 w 72 709 m 78 709 l S" # reaches a single character
|
|
pdf = build_styled_pdf([StyledPage([["short"], ["x"], ["end"]], ops=[rule, one])])
|
|
header, text = _derive_pdf(tree, pdf)
|
|
assert "~~" not in text and "markings" not in header
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_text_in_a_ruled_table_is_left_alone_and_its_grid_strikes_nothing(tree):
|
|
rows = [[f"cell {n}a ", f"cell {n}b"] for n in range(3)]
|
|
grid = ["0 G 0.5 w"]
|
|
for y in (736, 722, 708, 694, 680, 666): # horizontal rules through the lines of text as well as between
|
|
grid.append(f"60 {y - 4} m 200 {y - 4} l S")
|
|
for x in (60, 130, 200):
|
|
grid.append(f"{x} 660 m {x} 740 l S")
|
|
shaded = "1 1 0 rg 60 716 140 14 re f 0 g"
|
|
pdf = build_styled_pdf([StyledPage([[("cell 0a", {"bold": True}), " cell 0b"], ["cell 1a cell 1b"], ["cell 2a cell 2b"]], ops=["\n".join(grid), shaded])])
|
|
header, text = _derive_pdf(tree, pdf)
|
|
assert "*" not in text and "~" not in text and "markings" not in header
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_text_turned_by_a_quarter_is_struck_by_a_line_in_its_own_direction_only(tree):
|
|
pdf = build_styled_pdf([StyledPage([[("turned words", {"rotate": 90})]], ops=[
|
|
"0 G 0.5 w 60 700 m 300 700 l S", # a horizontal line is no strike-through for it
|
|
"0 G 0.7 w 75 720 m 75 756 l S", # nor is a line that is not in its middle
|
|
], size=(612, 1000))])
|
|
header, text = _derive_pdf(tree, pdf)
|
|
assert "~~" not in text
|
|
struck_pdf = build_styled_pdf([StyledPage([[("turned words", {"rotate": 90})]], ops=["0 G 0.7 w 68.8 720 m 68.8 792 l S"], size=(612, 1000))])
|
|
_header, text = _derive_pdf(tree, struck_pdf, "struck.pdf")
|
|
assert "~~turned words~~" in text
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_program_text_is_left_alone(tree):
|
|
code = [
|
|
[("if", {"bold": True, "color": BLUE}), " (x == 1) {"],
|
|
[" ", ("return", {"bold": True, "color": BLUE, "highlight": (1, 1, 0)}), " f(a, b);"],
|
|
[" ", ("y = g(x)", {"strike": True}), " + h[0];"],
|
|
["}"],
|
|
]
|
|
header, text = _derive_pdf(tree, build_styled_pdf([StyledPage(code)]))
|
|
assert "*" not in text and "~" not in text and "markings" not in header
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_shapes_that_recur_on_the_pages_are_part_of_the_page_not_of_the_text(tree, capsys):
|
|
rule = "0 G 0.7 w 72 723 m 160 723 l S"
|
|
banner = "1 1 0 rg 72 717 90 12 re f 0 g"
|
|
pages = [StyledPage([["Letterhead text"]], ops=[rule, banner], pictures=[Picture(400, 700, 40, 40, pixels=8)]) for _ in range(5)]
|
|
pages.append(StyledPage([["Last page"]], pictures=[Picture(300, 300, 60, 60, pixels=8, gray=0x40)]))
|
|
header, text = _derive_pdf(tree, build_styled_pdf(pages))
|
|
assert "*" not in text and "~" not in text and "markings" not in header
|
|
assert "Images on page(s) 6 are not" in capsys.readouterr().out
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_only_pictures_that_carry_content_are_named_in_the_check(tree, capsys):
|
|
page = lambda *pictures, ops=(): StyledPage([["x"]], pictures=list(pictures), ops=list(ops)) # noqa: E731
|
|
pdf = build_styled_pdf([
|
|
page(Picture(0, 0, 612, 792, pixels=8, gray=0x20)), # 1: a background
|
|
page(Picture(100, 400, 40, 40, pixels=2)), # 2: a filler a few pixels in size
|
|
page(Picture(100, 400, 6, 6, pixels=8)), # 3: a speck a few points in size
|
|
page(Picture(100, 400, 120, 90, pixels=8, gray=0x40)), # 4: content
|
|
page(ops=[f"0 G 0.5 w {80 + 8 * n} 400 m {84 + 8 * n} {430 + (n % 3) * 9} l S" for n in range(30)]), # 5: a drawing
|
|
])
|
|
header, _text = _derive_pdf(tree, pdf)
|
|
assert "CHECK Images on page(s) 4-5 are not contained in the text" in capsys.readouterr().out
|
|
assert header["images"] == "4-5"
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_markings_nest_and_a_gap_between_two_equal_runs_is_inside_the_tag(tree):
|
|
pdf = build_styled_pdf([StyledPage([["Running text, ", ("two words", {"highlight": YELLOW, "strike": True, "italic": True}), " and the rest of the line."]])])
|
|
_header, text = _derive_pdf(tree, pdf)
|
|
assert "**~~*two words*~~**" in text
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_a_marking_never_runs_over_a_line_or_a_page_marker(tree):
|
|
pdf = build_styled_pdf([
|
|
StyledPage([[("first line", {"highlight": YELLOW})], [("second line", {"highlight": YELLOW})]]),
|
|
StyledPage([[("third page", {"highlight": YELLOW, "strike": True})]]),
|
|
])
|
|
_header, text = _derive_pdf(tree, pdf)
|
|
for line in text.splitlines():
|
|
assert line.count("**") % 2 == 0 and line.count("~~") % 2 == 0, line
|
|
assert "**first line**\n**second line**" in text
|
|
assert "**\n\n<!-- page 2 -->\n**" in text
|
|
|
|
|
|
def _tags_hug_their_text(text: str) -> bool:
|
|
"""No opening tag is followed, and no closing tag preceded, by whitespace."""
|
|
open_kinds: set[str] = set()
|
|
for match in re.finditer(r"\*\*|~~|\*", text):
|
|
kind = match.group()
|
|
if kind in open_kinds:
|
|
open_kinds.discard(kind)
|
|
if text[match.start() - 1].isspace():
|
|
return False
|
|
else:
|
|
open_kinds.add(kind)
|
|
if text[match.end():match.end() + 1].isspace():
|
|
return False
|
|
return True
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_no_tag_ever_sits_next_to_whitespace_on_its_inner_side(tree):
|
|
pdf = build_styled_pdf([StyledPage([
|
|
["Plain ", ("yellow ", {"highlight": YELLOW}), "and ", (" padded", {"highlight": GREEN}), " words ", ("gone ", {"strike": True}), "on."],
|
|
[("Running text that mixes ", {}), ("bold ", {"bold": True}), (" and ", {"bold": True}), ("more bold", {"bold": True}), " then plain text."],
|
|
[(" ", {"strike": True}), ("left", {"strike": True}), (" ", {"strike": True}), ("right", {"strike": True})],
|
|
])])
|
|
_header, text = _derive_pdf(tree, pdf)
|
|
assert "**" in text and "~~" in text
|
|
assert _tags_hug_their_text(text), text
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_removing_the_markings_gives_the_text_the_pdf_has_without_them(tree):
|
|
styled = build_styled_pdf([StyledPage([
|
|
["Plain ", ("yellow", {"highlight": YELLOW}), " ", ("red", {"color": RED}), " ", ("struck", {"strike": True}), " end of the line, ", ("bold", {"bold": True}), " ", ("slanted", {"italic": True}), "."],
|
|
[("whole line", {"highlight": GREEN})],
|
|
])])
|
|
unstyled = build_styled_pdf([StyledPage([["Plain yellow red struck end of the line, bold slanted."], ["whole line"]])])
|
|
_header, marked = _derive_pdf(tree, styled, "marked.pdf")
|
|
_header, plain = _derive_pdf(tree, unstyled, "plain.pdf")
|
|
assert marked != plain
|
|
assert _markings_stripped(marked) == plain
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_the_legend_is_at_most_three_lines(tree):
|
|
pdf = build_styled_pdf([StyledPage([[
|
|
"A line of running text with ", ("highlight", {"highlight": YELLOW}), ", ", ("struck", {"strike": True}), " and an ",
|
|
("italic", {"italic": True}), " word, and ", ("bold", {"bold": True}), ".",
|
|
]])])
|
|
header, _text = _derive_pdf(tree, pdf)
|
|
assert [entry.split(" | ")[0] for entry in header["markings"]] == ["**…**", "~~…~~", "*…*"]
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_pages_with_images_get_one_check_line(tree, capsys):
|
|
pdf = build_styled_pdf([StyledPage([["a"]]), StyledPage([["b"]], image=True), StyledPage([["c"]], image=True), StyledPage([["d"]])])
|
|
header, _text = _derive_pdf(tree, pdf)
|
|
out = capsys.readouterr().out
|
|
assert out.count("CHECK") == 2 and "Images on page(s) 2-3 are not contained in the text" in out
|
|
assert header["images"] == "2-3"
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_a_page_whose_text_does_not_match_the_layout_is_left_unmarked_with_a_warning(monkeypatch, tree, capsys):
|
|
pdf = build_styled_pdf([StyledPage([[("highlighted", {"highlight": YELLOW})]]), StyledPage([[("fine", {"highlight": YELLOW})]])])
|
|
real = pdf_layout.analyse
|
|
|
|
def disagree(data):
|
|
first, second = real(data)
|
|
garbled = tuple(dataclasses.replace(char, text="z") for char in first.chars)
|
|
return [dataclasses.replace(first, chars=garbled), second]
|
|
|
|
monkeypatch.setattr(pdf_layout, "analyse", disagree)
|
|
header, text = _derive_pdf(tree, pdf)
|
|
out = capsys.readouterr().out
|
|
assert "highlighted" in text and "**highlighted" not in text
|
|
assert "**fine**" in text
|
|
assert "WARN" in out and "Page(s) 1" in out and "left out" in out
|
|
assert str(header["unmatched"]) == "1" and "images" not in header
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_a_layout_that_cannot_be_read_costs_the_markings_not_the_derivation(monkeypatch, tree, capsys):
|
|
def broken(data):
|
|
raise ValueError("boom")
|
|
|
|
monkeypatch.setattr(pdf_layout, "analyse", broken)
|
|
header, text = _derive_pdf(tree, build_styled_pdf([StyledPage([[("kept", {"highlight": YELLOW})]])], outline=[("Top", 1, 0)]))
|
|
out = capsys.readouterr().out
|
|
assert "kept" in text and "**" not in text and header["outline"][0].startswith("Top | level 1 | page 1")
|
|
assert "images" not in header and "unmatched" not in header
|
|
assert "WARN" in out and "boom" in out
|
|
|
|
|
|
|
|
def test_a_pdf_without_pdfminer_is_refused_by_package_name(tree, monkeypatch, capsys):
|
|
monkeypatch.setattr(type(filters.filter_for("a.pdf")), "missing", lambda self: ["pdfminer.six"])
|
|
_put(tree, "a.pdf", b"%PDF-1.4")
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/a.pdf")
|
|
assert "pdfminer.six" in capsys.readouterr().out
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
def test_the_filter_names_packages_by_their_installable_name(monkeypatch):
|
|
real_import = importlib.import_module
|
|
|
|
def refuse(name):
|
|
if name == "pdfminer":
|
|
raise ImportError(name)
|
|
return real_import(name)
|
|
|
|
monkeypatch.setattr(importlib, "import_module", refuse)
|
|
assert filters.filter_for("a.pdf").missing() == ["pdfminer.six"]
|
|
|
|
|
|
@needs_pypdf
|
|
@needs_pdfminer
|
|
def test_the_engine_line_names_both_pdf_libraries(tree):
|
|
header, _text = _derive_pdf(tree, build_styled_pdf([StyledPage([["x"]])]))
|
|
assert "pypdf " in header["engine"] and "pdfminer.six " in header["engine"]
|
|
|
|
|
|
def test_outline_lines_count_the_header_whatever_fields_it_has():
|
|
derivation = filters.Derivation(
|
|
text="<!-- page 1 -->\nbody\n\n<!-- page 2 -->\nmore",
|
|
fields=(("pages", "2"),),
|
|
outline=(filters.OutlineEntry("Second", 1, 2, 4),),
|
|
)
|
|
for original_name in (None, "Renamed.pdf"):
|
|
content = filters.render(filters.filter_for("a.pdf"), derivation, derived_from="x", original_name=original_name)
|
|
lines = content.decode("utf-8").split("\n")
|
|
stated = next(line for line in lines if line.startswith("outline:"))
|
|
assert lines[int(re.search(r"line (\d+)", stated).group(1)) - 1] == "<!-- page 2 -->"
|
|
|
|
|
|
def test_characters_are_matched_across_a_difference_in_the_middle():
|
|
flag = (True, None, None)
|
|
glyphs = [pdf_layout.Glyph(c, i >= 4, False, False) for i, c in enumerate("start ghost middle end")]
|
|
found, coverage = pdf_layout._align("start middle end", glyphs, [(True if g.strong else None, None, None) for g in glyphs])
|
|
assert coverage == 1.0
|
|
assert [found.get(i) for i in (0, 6, 7, 13)] == [(None, None, None), flag, flag, flag]
|
|
|
|
|
|
def test_page_ranges_join_neighbours():
|
|
assert pdf_layout.page_ranges([6, 7, 8, 10, 12, 13, 8]) == "6-8, 10, 12-13"
|
|
assert pdf_layout.page_ranges([]) == ""
|
|
|
|
|
|
def test_colours_are_read_from_gray_rgb_and_cmyk():
|
|
assert pdf_layout._hex(0.5) == "#808080"
|
|
assert pdf_layout._hex((1.0, 0.0, 0.0)) == "#ff0000"
|
|
assert pdf_layout._hex((0.0, 0.0, 1.0, 0.0)) == "#ffff00"
|
|
assert pdf_layout._hex(("pattern",)) is None and pdf_layout._hex(None) is None
|
|
|
|
|
|
# --- Outlook .msg --------------------------------------------------------------
|
|
|
|
|
|
SENT = 134_000_000_000_000_000 # 2025-08-18T14:13:20+00:00
|
|
|
|
|
|
@needs_olefile
|
|
def test_a_msg_becomes_a_folder_like_a_mail(tree, capsys):
|
|
data = [b"attachment bytes" * 400, b"small"]
|
|
original = build_msg(
|
|
subject="Unterlagen", body="Hello Bob\r\nSee attached.", sender="Anna", sender_address="anna@example.org",
|
|
to="bob@example.org; carol@example.org", sent=SENT,
|
|
attachments=[("report.bin", data[0]), ("note.txt", data[1])],
|
|
)
|
|
_put(tree, "Mail.msg", original)
|
|
_derive("incoming/Mail.msg")
|
|
out = capsys.readouterr().out
|
|
|
|
folder = tree / "incoming/Mail"
|
|
assert (folder / "original.msg").read_bytes() == original
|
|
assert (folder / "report.bin").read_bytes() == data[0]
|
|
assert (folder / "note.txt").read_bytes() == data[1]
|
|
header, text = _header_and_text(folder / "derived.md")
|
|
assert header["filter"] == "msg" and "olefile" in header["engine"]
|
|
assert header["from"] == "Anna <anna@example.org>"
|
|
assert header["to"] == ["bob@example.org", "carol@example.org"]
|
|
assert header["subject"] == "Unterlagen" and str(header["date"]).startswith("2025-08-18")
|
|
assert header["attachments"] == ["report.bin", "note.txt"]
|
|
assert text.strip() == "Hello Bob\nSee attached."
|
|
assert "report.bin" in out
|
|
|
|
|
|
@needs_olefile
|
|
def test_a_msg_with_only_html_takes_the_html_text(tree):
|
|
_put(tree, "h.msg", build_msg(subject="S", html="<html><body><p>Only <b>html</b> here</p></body></html>"))
|
|
_derive("incoming/h.msg")
|
|
_header, text = _header_and_text(tree / "incoming/h/derived.md")
|
|
assert "Only" in text and "html here" in text and "<p>" not in text
|
|
|
|
|
|
_HTML_BODY = "<html><body><p>" + "Real content of the mail. " * 20 + "</p></body></html>"
|
|
|
|
|
|
@needs_olefile
|
|
def test_a_msg_with_a_placeholder_plain_text_takes_the_html_text_and_warns(tree, capsys):
|
|
_put(tree, "p.msg", build_msg(subject="S", body="Plain text version not available", html=_HTML_BODY))
|
|
_derive("incoming/p.msg")
|
|
out = capsys.readouterr().out
|
|
_header, text = _header_and_text(tree / "incoming/p/derived.md")
|
|
|
|
assert "Real content of the mail." in text and "not available" not in text
|
|
assert "The plain text (32 characters) is under 10% of the HTML text" in out
|
|
|
|
|
|
@needs_olefile
|
|
def test_a_msg_with_a_good_plain_text_keeps_it_without_a_warning(tree, capsys):
|
|
plain = "Real content of the mail. " * 18
|
|
_put(tree, "g.msg", build_msg(subject="S", body=plain, html=_HTML_BODY))
|
|
_derive("incoming/g.msg")
|
|
out = capsys.readouterr().out
|
|
_header, text = _header_and_text(tree / "incoming/g/derived.md")
|
|
|
|
assert text.strip() == plain.strip()
|
|
assert "is under" not in out
|
|
|
|
|
|
@needs_olefile
|
|
def test_a_msg_with_only_rtf_warns_and_has_no_text(tree, capsys):
|
|
_put(tree, "r.msg", build_msg(subject="S", rtf=b"{\\rtf1 hello}"))
|
|
_derive("incoming/r.msg")
|
|
out = capsys.readouterr().out
|
|
assert "WARN" in out and "RTF" in out
|
|
_header, text = _header_and_text(tree / "incoming/r/derived.md")
|
|
assert text.strip() == ""
|
|
|
|
|
|
@needs_olefile
|
|
def test_a_msg_names_inline_images_and_embedded_messages_without_writing_them(tree, capsys):
|
|
_put(tree, "m.msg", build_msg(
|
|
subject="S", body="text", inline_images=[("logo.png", b"png")], embedded=["forwarded.msg"],
|
|
attachments=[("real.bin", b"x")],
|
|
))
|
|
_derive("incoming/m.msg")
|
|
out = capsys.readouterr().out
|
|
assert sorted(p.name for p in (tree / "incoming/m").iterdir()) == ["derived.md", "original.msg", "real.bin"]
|
|
assert "inline image" in out and "forwarded.msg" in out
|
|
|
|
|
|
@needs_olefile
|
|
def test_a_msg_attached_to_a_mail_is_quoted(tree, capsys):
|
|
inner = build_msg(subject="Inner subject", body="Inner words", sender="Dora", to="x@example.org")
|
|
_put(tree, "mail.eml", make_mail([("inner.msg", inner, "application/vnd.ms-outlook")]))
|
|
_derive("incoming/mail.eml")
|
|
folder = tree / "incoming/mail"
|
|
assert sorted(p.name for p in folder.iterdir()) == ["derived.md", "original.eml"]
|
|
_header, text = _header_and_text(folder / "derived.md")
|
|
assert "## Attached message: Inner subject" in text and "> Inner words" in text
|
|
|
|
|
|
@needs_olefile
|
|
def test_a_file_that_is_not_a_msg_is_refused_untouched(tree):
|
|
_put(tree, "x.msg", b"plain text pretending to be a message")
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/x.msg")
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
@needs_olefile
|
|
def test_a_compound_file_that_is_not_a_message_is_refused(tree):
|
|
from chemenu.tests.binary_formats import build_cfb
|
|
|
|
_put(tree, "x.msg", build_cfb({"WordDocument": b"abc"}))
|
|
_refused("incoming/x.msg")
|
|
assert not (tree / "incoming/x").exists()
|
|
|
|
|
|
def test_a_msg_without_the_package_is_refused_and_a_docx_still_works(tree, monkeypatch, capsys):
|
|
monkeypatch.setattr(type(filters.filter_for("a.msg")), "missing", lambda self: ["olefile"])
|
|
_put(tree, "a.msg", b"\xd0\xcf\x11\xe0")
|
|
before = _snapshot(tree / "incoming")
|
|
_refused("incoming/a.msg")
|
|
assert "olefile" in capsys.readouterr().out
|
|
assert _snapshot(tree / "incoming") == before
|
|
|
|
|
|
# --- doctor ---------------------------------------------------------------------
|
|
|
|
|
|
def test_doctor_names_each_filter_that_needs_a_package():
|
|
names = {check.name for check in doctor.check_import_filters()}
|
|
assert names == {"import-filter-pdf", "import-filter-msg"}
|
|
|
|
|
|
def test_doctor_warns_for_a_missing_package_and_never_fails(monkeypatch):
|
|
monkeypatch.setattr(type(filters.filter_for("a.pdf")), "missing", lambda self: ["pypdf"])
|
|
checks = {check.name: check for check in doctor.check_import_filters()}
|
|
assert checks["import-filter-pdf"].status == "WARN"
|
|
assert "pypdf" in checks["import-filter-pdf"].detail and checks["import-filter-pdf"].fix
|
|
assert all(check.status != "FAIL" for check in checks.values())
|
|
|
|
|
|
@needs_pypdf
|
|
def test_doctor_reports_an_installed_package_with_its_version():
|
|
check = next(c for c in doctor.check_import_filters() if c.name == "import-filter-pdf")
|
|
assert check.status == "OK" and check.detail.startswith("pypdf ") and ".pdf" in check.detail
|