From a829560bf210173b2a687fb57289a31915bb0647 Mon Sep 17 00:00:00 2001 From: Torben Nehmer Date: Wed, 14 Oct 2026 05:17:50 +0200 Subject: [PATCH] raw derive PDF: Gliederung mit Seite und Zeile im Kopf, Hinterlegungen, Durchstreichungen und abweichende Textfarben als Auszeichnung im Text, Bildseiten als CHECK (#225); 10.1.0-beta.1 Files changed: - CHANGES.md - VERSION - docs/reference/commands/raw-material-and-uploads.md - raw/CONTRACT.md - tools/CONTRACT.md - tools/README.md - tools/chemenu/commands/raw/derive.py - tools/chemenu/filters/__init__.py - tools/chemenu/filters/pdf.py - tools/chemenu/filters/pdf_layout.py - tools/chemenu/tests/binary_formats.py - tools/chemenu/tests/test_raw_derive_binary.py - tools/requirements.txt Co-Authored-By: Claude Code Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2 --- CHANGES.md | 14 + VERSION | 2 +- .../commands/raw-material-and-uploads.md | 3 +- raw/CONTRACT.md | 13 + tools/CONTRACT.md | 3 +- tools/README.md | 2 +- tools/chemenu/commands/raw/derive.py | 21 +- tools/chemenu/filters/__init__.py | 36 +- tools/chemenu/filters/pdf.py | 74 +++- tools/chemenu/filters/pdf_layout.py | 318 ++++++++++++++++++ tools/chemenu/tests/binary_formats.py | 134 +++++++- tools/chemenu/tests/test_raw_derive_binary.py | 261 +++++++++++++- tools/requirements.txt | 1 + 13 files changed, 860 insertions(+), 22 deletions(-) create mode 100644 tools/chemenu/filters/pdf_layout.py diff --git a/CHANGES.md b/CHANGES.md index 2deac09..bad6d2c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,20 @@ place only: `instructions/dev/version-parts.md` § The candidate model. --- +## 10.1.0-beta.1 - 2026-10-13 - raw derive PDF: Gliederung mit Seite und Zeile im Kopf, Hinterlegungen, Durchstreichungen und abweichende Textfarben als Auszeichnung im Text, Bildseiten als CHECK + +**Author:** Torben Nehmer + + +- raw derive PDF: Gliederung mit Seite und Zeile im Kopf, Hinterlegungen, Durchstreichungen und abweichende Textfarben als Auszeichnung im Text, Bildseiten als CHECK + + +### Große PDFs: Gliederung und Auszeichnungen + +Bei einem PDF schreibt `raw derive` die Gliederung als Feld `outline` in den Kopf, je Eintrag mit Ebene, Seite und der Zeile der Seitenmarke in `derived.md`; so findet ein Leser einen Abschnitt, ohne das ganze Dokument zu lesen. Im Text stehen drei Auszeichnungen für das, was die Seite nur durch ihre Form sagt: `…` für eine Hinterlegung, `~~…~~` für eine Durchstreichung und `*…*` für eine vom Fließtext abweichende Textfarbe ohne Link. Sie geben das Aussehen wieder, nicht die Bedeutung; ohne sie ist der Text derselbe wie zuvor. Das Feld `markings` im Kopf ist die Legende mit Anzahl und Seiten, kein Tag überschreitet eine Zeile oder Seitenmarke, und Bildseiten melden einen CHECK. Das Layout liest `pdfminer.six`, das neu in `requirements.txt` steht; ein PDF ohne Gliederung und Auszeichnungen wird wie bisher abgeleitet, vorhandene `derived.md` bleiben unberührt. + +--- + ## 10.0.0 - 2026-10-13 - Einstellungen in drei Stufen, Eingangskanäle, raw fetch mit Site-Handlern und Login, Backup und Exporte **Author:** Torben Nehmer diff --git a/VERSION b/VERSION index a13e7b9..73295a5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -10.0.0 +10.1.0-beta.1 diff --git a/docs/reference/commands/raw-material-and-uploads.md b/docs/reference/commands/raw-material-and-uploads.md index c445cb6..70596d0 100644 --- a/docs/reference/commands/raw-material-and-uploads.md +++ b/docs/reference/commands/raw-material-and-uploads.md @@ -192,7 +192,8 @@ tools/wikitool raw derive incoming/scans.zip - A single file directly in `incoming/` becomes a folder named after its stem (or `--name`) holding `original.` - the original bytes, moved - and `derived.md`. The fixed names keep the path inside the path budget; the header of `derived.md` records the file's own name as `original_name`. - A folder is derived in place: each file with a filter gets `.md` beside it, nothing is renamed, and the files without a filter are named in the output. - `derived.md` starts with a fixed header (`derived_by`, `filter`, `engine`, `derived`, `derived_from`, `original_name` for a renamed file, `title`, then the fields of the format) followed by the derived text. The same bytes always give the same text; only `derived` differs between runs. `engine` names the tool version and, where a package reads the format, its version. -- Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, `.xlsm`), e-mail (`.eml`) and ZIP archives (`.zip`), read with the standard library only; PDF (`.pdf`, `pypdf`) and Outlook messages (`.msg`, `olefile`), each read by a package in `tools/requirements.txt` that is imported only when such a file is derived. A missing package refuses that format alone, with the preflight hint; `wikitool doctor` shows one `import-filter-` line per package-backed format. A PDF gets `` markers and a WARN when it yields almost no text (a scan; there is no OCR); an encrypted PDF is refused. A `.msg` is a container like an `.eml`; an embedded message inside it is named, not read. The text of a mail (`.eml` and `.msg` alike) is its plain text, unless that is under 10% of the length of the HTML text - a "plain text version not available" placeholder - in which case it is the HTML text and a WARN names both lengths. The other version of the text in a `multipart/alternative` is not an attachment. +- Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, `.xlsm`), e-mail (`.eml`) and ZIP archives (`.zip`), read with the standard library only; PDF (`.pdf`, `pypdf` and `pdfminer.six`) and Outlook messages (`.msg`, `olefile`), each read by packages in `tools/requirements.txt` that are imported only when such a file is derived. A missing package refuses that format alone, with the preflight hint; `wikitool doctor` shows one `import-filter-` line per package-backed format. A PDF gets `` markers and a WARN when it yields almost no text (a scan; there is no OCR); an encrypted PDF is refused. A `.msg` is a container like an `.eml`; an embedded message inside it is named, not read. The text of a mail (`.eml` and `.msg` alike) is its plain text, unless that is under 10% of the length of the HTML text - a "plain text version not available" placeholder - in which case it is the HTML text and a WARN names both lengths. The other version of the text in a `multipart/alternative` is not an attachment. +- A PDF also gets, where the document has them: an `outline` header field (one entry per outline item: title, level, page and the line of `derived.md` where that page's marker is), and markings in the text that follow the visible form only - `…` for text over a coloured background, `*…*` for text in a colour other than the running text that is not under a link, `~~…~~` for strike-through. A `markings` header field is the legend (form, colour, places, pages); removing the markings gives the text the PDF has without them. A CHECK names the pages with images, which the text does not contain. A PDF without outline, markings and images is derived as before. - A workbook (`.xlsx`, `.xlsm`) gets its cells as files: beside its `.md` a folder `.sheets/` (`derived.sheets/` for a single file) holds one `NN-.tsv` per sheet - tab-separated per RFC 4180, UTF-8, row 1 of the file is row 1 of the sheet - and, for a sheet with formulas, `NN-.formulas.tsv` listing cell, formula and last stored value. The `.md` is the overview: one section per sheet with its range, the file names and a preview. Formulas are not recalculated; a shared formula is written out per cell, and one that cannot be is named `shared:` with a WARN. The folder counts against the path budget and is accepted with the rest; an existing one is never overwritten. - A ZIP and a mail are containers. A ZIP at the top of `incoming/` becomes the folder `incoming//` holding its files under their own names, each with a `.md` where a filter reads it; the ZIP is removed and the output names it with its sha256. A mail becomes a folder with `original.eml`, `derived.md` and its attachments under their own names; a ZIP attached to it is unpacked once into a folder of its own, and an attached mail is quoted in `derived.md`, not unpacked. A ZIP inside a ZIP, an absolute or `..` path, a symbolic link, an encrypted entry, more than 1000 entries, a file over 25 MiB or more than 250 MiB in total refuse the whole call. Hidden entries (`__MACOSX/`, `.DS_Store`, anything starting with a dot) are named, not written. A name taken twice, or reserved (`original.*` and `derived.md` at the top of a mail's folder, `_capture.json` anywhere), gets `-2`, `-3` before its extension, and the output lists each rename. - A name already taken under `raw/` is a WARN, not a refusal: whether the folder is a later edition of that source or a separate one is the human's call. A later edition of a derived source is replaced file by file with `raw accept --replaces`; there is no `--replaces-bundle` for it. diff --git a/raw/CONTRACT.md b/raw/CONTRACT.md index 1974490..f4c7399 100644 --- a/raw/CONTRACT.md +++ b/raw/CONTRACT.md @@ -391,6 +391,19 @@ the file and does not recalculate it. Text that is data is read by a script, not eye; the overview is what tells the reader which file to open. The folder is part of the derivation, accepted with it, and an existing one is never overwritten. +**A PDF's outline and markings are in the derivation.** The header of `derived.md` lists the +PDF's outline as `outline` entries - title, level, page and the line of that page's +`` marker in the file - so a reader finds a section without reading the whole +text. In the text, three markings carry what the page shows by form alone: a background fill is +`…`, a strike-through is `~~…~~`, and text in a colour other than the +running text, without a link under it, is `*…*`. A marking records how the page looks, not what +it means: nothing in the tool decides that yellow is a deadline, and removing the markings gives +the same text as without them. The `markings` header field is the legend - each marking in use, +with its count and pages. No tag crosses a line or a page marker. A page whose text cannot be +matched to its layout gets no markings and a WARN; a PDF with images the text does not hold gets +a CHECK naming the pages, because the tool has no OCR. A PDF without outline or markings is +derived as before. Reading the layout needs `pdfminer.six` beside `pypdf`. + **The original stays.** The derivation is the tool's reading of it, never the source: with the original kept, a claim in `kb/` can still be checked against what the document said when a derivation dropped something, and a later version of the tool can derive it better. Both go into diff --git a/tools/CONTRACT.md b/tools/CONTRACT.md index d3e4731..777e7ea 100644 --- a/tools/CONTRACT.md +++ b/tools/CONTRACT.md @@ -1607,7 +1607,8 @@ Derive reading text from a PDF, e-mail, Office file or web page in `incoming/`, - A single file directly in `incoming/` becomes a folder named after its stem (or `--name`) holding `original.` - the original bytes, moved - and `derived.md`. The fixed names keep the path inside the path budget; the header of `derived.md` records the file's own name as `original_name`. - A folder is derived in place: each file with a filter gets `.md` beside it, nothing is renamed, and the files without a filter are named in the output. - `derived.md` starts with a fixed header (`derived_by`, `filter`, `engine`, `derived`, `derived_from`, `original_name` for a renamed file, `title`, then the fields of the format) followed by the derived text. The same bytes always give the same text; only `derived` differs between runs. `engine` names the tool version and, where a package reads the format, its version. -- Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, `.xlsm`), e-mail (`.eml`) and ZIP archives (`.zip`), read with the standard library only; PDF (`.pdf`, `pypdf`) and Outlook messages (`.msg`, `olefile`), each read by a package in `tools/requirements.txt` that is imported only when such a file is derived. A missing package refuses that format alone, with the preflight hint; `wikitool doctor` shows one `import-filter-` line per package-backed format. A PDF gets `` markers and a WARN when it yields almost no text (a scan; there is no OCR); an encrypted PDF is refused. A `.msg` is a container like an `.eml`; an embedded message inside it is named, not read. The text of a mail (`.eml` and `.msg` alike) is its plain text, unless that is under 10% of the length of the HTML text - a "plain text version not available" placeholder - in which case it is the HTML text and a WARN names both lengths. The other version of the text in a `multipart/alternative` is not an attachment. +- Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, `.xlsm`), e-mail (`.eml`) and ZIP archives (`.zip`), read with the standard library only; PDF (`.pdf`, `pypdf` and `pdfminer.six`) and Outlook messages (`.msg`, `olefile`), each read by packages in `tools/requirements.txt` that are imported only when such a file is derived. A missing package refuses that format alone, with the preflight hint; `wikitool doctor` shows one `import-filter-` line per package-backed format. A PDF gets `` markers and a WARN when it yields almost no text (a scan; there is no OCR); an encrypted PDF is refused. A `.msg` is a container like an `.eml`; an embedded message inside it is named, not read. The text of a mail (`.eml` and `.msg` alike) is its plain text, unless that is under 10% of the length of the HTML text - a "plain text version not available" placeholder - in which case it is the HTML text and a WARN names both lengths. The other version of the text in a `multipart/alternative` is not an attachment. +- A PDF also gets, where the document has them: an `outline` header field (one entry per outline item: title, level, page and the line of `derived.md` where that page's marker is), and markings in the text that follow the visible form only - `…` for text over a coloured background, `*…*` for text in a colour other than the running text that is not under a link, `~~…~~` for strike-through. A `markings` header field is the legend (form, colour, places, pages); removing the markings gives the text the PDF has without them. A CHECK names the pages with images, which the text does not contain. A PDF without outline, markings and images is derived as before. - A workbook (`.xlsx`, `.xlsm`) gets its cells as files: beside its `.md` a folder `.sheets/` (`derived.sheets/` for a single file) holds one `NN-.tsv` per sheet - tab-separated per RFC 4180, UTF-8, row 1 of the file is row 1 of the sheet - and, for a sheet with formulas, `NN-.formulas.tsv` listing cell, formula and last stored value. The `.md` is the overview: one section per sheet with its range, the file names and a preview. Formulas are not recalculated; a shared formula is written out per cell, and one that cannot be is named `shared:` with a WARN. The folder counts against the path budget and is accepted with the rest; an existing one is never overwritten. - A ZIP and a mail are containers. A ZIP at the top of `incoming/` becomes the folder `incoming//` holding its files under their own names, each with a `.md` where a filter reads it; the ZIP is removed and the output names it with its sha256. A mail becomes a folder with `original.eml`, `derived.md` and its attachments under their own names; a ZIP attached to it is unpacked once into a folder of its own, and an attached mail is quoted in `derived.md`, not unpacked. A ZIP inside a ZIP, an absolute or `..` path, a symbolic link, an encrypted entry, more than 1000 entries, a file over 25 MiB or more than 250 MiB in total refuse the whole call. Hidden entries (`__MACOSX/`, `.DS_Store`, anything starting with a dot) are named, not written. A name taken twice, or reserved (`original.*` and `derived.md` at the top of a mail's folder, `_capture.json` anywhere), gets `-2`, `-3` before its extension, and the output lists each rename. - A name already taken under `raw/` is a WARN, not a refusal: whether the folder is a later edition of that source or a separate one is the human's call. A later edition of a derived source is replaced file by file with `raw accept --replaces`; there is no `--replaces-bundle` for it. diff --git a/tools/README.md b/tools/README.md index 7f7c8e9..3f8005a 100644 --- a/tools/README.md +++ b/tools/README.md @@ -69,7 +69,7 @@ excepted. `jsonschema` and `PyYAML` are hard dependencies, not optional extras: schema validation is the tool's whole safety net, so `cli.py` fails loudly with the -fix rather than degrading silently. `pypdf` and `olefile` are in `requirements.txt` too +fix rather than degrading silently. `pypdf`, `pdfminer.six` and `olefile` are in `requirements.txt` too but are imported only when a PDF or an Outlook `.msg` is derived, so a missing one stops that format alone; `doctor` reports each. diff --git a/tools/chemenu/commands/raw/derive.py b/tools/chemenu/commands/raw/derive.py index 9b84980..08d81b1 100644 --- a/tools/chemenu/commands/raw/derive.py +++ b/tools/chemenu/commands/raw/derive.py @@ -358,17 +358,26 @@ def _derive_next(target: Path) -> str: "reads the format, its version.", "Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, " "`.xlsm`), e-mail (`.eml`) and ZIP archives (`.zip`), read with the standard library " - "only; PDF (`.pdf`, `pypdf`) and Outlook messages (`.msg`, `olefile`), each read by a " - "package in `tools/requirements.txt` that is imported only when such a file is derived. " - "A missing package refuses that format alone, with the preflight hint; `wikitool " - "doctor` shows one `import-filter-` line per package-backed format. A PDF gets " - "`` markers and a WARN when it yields almost no text (a scan; there is " - "no OCR); an encrypted PDF is refused. A `.msg` is a container like an `.eml`; an " + "only; PDF (`.pdf`, `pypdf` and `pdfminer.six`) and Outlook messages (`.msg`, `olefile`), each " + "read by packages in `tools/requirements.txt` that are imported only when such a file is " + "derived. A missing package refuses that format alone, with the preflight hint; " + "`wikitool doctor` shows one `import-filter-` line per package-backed format. A " + "PDF gets `` markers and a WARN when it yields almost no text (a scan; " + "there is no OCR); an encrypted PDF is refused. A `.msg` is a container like an `.eml`; an " "embedded message inside it is named, not read. The text of a mail (`.eml` and `.msg` " "alike) is its plain text, unless that is under 10% of the length of the HTML text - a " "\"plain text version not available\" placeholder - in which case it is the HTML text " "and a WARN names both lengths. The other version of the text in a " "`multipart/alternative` is not an attachment.", + "A PDF also gets, where the document has them: an `outline` header field (one entry per " + "outline item: title, level, page and the line of `derived.md` where that page's " + "marker is), and markings in the text that follow the visible form only - `\">…` for text over a coloured background, `*…*` for text in a " + "colour other than the running text that is not under a link, `~~…~~` for strike-through. " + "A `markings` header field is the legend (form, colour, places, pages); removing the " + "markings gives the text the PDF has without them. A CHECK names the pages with images, " + "which the text does not contain. A PDF without outline, markings and images is derived " + "as before.", "A workbook (`.xlsx`, `.xlsm`) gets its cells as files: beside its `.md` a folder " "`.sheets/` (`derived.sheets/` for a single file) holds one `NN-.tsv` " "per sheet - tab-separated per RFC 4180, UTF-8, row 1 of the file is row 1 of the sheet " diff --git a/tools/chemenu/filters/__init__.py b/tools/chemenu/filters/__init__.py index 738e8a2..1b70693 100644 --- a/tools/chemenu/filters/__init__.py +++ b/tools/chemenu/filters/__init__.py @@ -26,19 +26,33 @@ MAX_CHECKS = 3 COMPANION_SUFFIX = ".sheets" COMPANION_DIR = "\x00companion\x00" # NUL cannot occur in XML text: a safe placeholder inside `Derivation.text` +DISTRIBUTIONS = {"pdfminer": "pdfminer.six"} # import name -> package name, where they differ + PREFLIGHT_HINT = ( "Run the preflight to install it (tools/preflight.sh, or tools/preflight.ps1 on Windows; " "instructions/preflight.md). Every other format keeps working." ) +@dataclass(frozen=True) +class OutlineEntry: + """One entry of a document's outline. `line` is the line of `Derivation.text` + (1-based) the entry points at; `render` turns it into a line of the file.""" + + title: str + level: int + page: Optional[int] = None + line: Optional[int] = None + + @dataclass(frozen=True) class Derivation: """What a filter makes of one document. `fields` are the format-specific header lines, after `title`; a value is a - string or a list of strings. `checks` are the short English lines `raw derive` - prints after a success - already narrowed to the case that occurred. + string or a list of strings. `outline` becomes the last header field, with the + lines counted in the finished file. `checks` are the short English lines + `raw derive` prints after a success - already narrowed to the case that occurred. `files` are further files of the derivation (a spreadsheet's sheets), as `(name, bytes)`. They are written into a folder beside the `.md`, named @@ -52,6 +66,7 @@ class Derivation: warnings: tuple[str, ...] = () checks: tuple[str, ...] = () files: tuple[tuple[str, bytes], ...] = () + outline: tuple[OutlineEntry, ...] = () def __post_init__(self) -> None: if len(self.checks) > MAX_CHECKS: @@ -105,12 +120,13 @@ class Filter: return True def missing(self) -> list[str]: + """The packages (by their installable name) this filter cannot import.""" missing = [] for package in self.requires: try: importlib.import_module(package) except ImportError: - missing.append(package) + missing.append(DISTRIBUTIONS.get(package, package)) return missing def engine(self) -> str: @@ -119,6 +135,7 @@ class Filter: except VersionError: parts = ["wikitool unknown"] for package in self.requires: + package = DISTRIBUTIONS.get(package, package) try: parts.append(f"{package} {importlib.metadata.version(package)}") except importlib.metadata.PackageNotFoundError: @@ -206,10 +223,23 @@ def render( if companion is None: raise ValueError("a derivation with files needs the name of the folder they go into") text = text.replace(COMPANION_DIR, companion) + if derivation.outline: + # the header's length does not depend on the outline's content: one field, one line + first_line = web_capture.header(fields + [("outline", [""])]).count("\n") + 2 + fields.append(("outline", [_outline_line(entry, first_line) for entry in derivation.outline])) content = web_capture.header(fields) + "\n" + (text + "\n" if text else "") return content.encode("utf-8") +def _outline_line(entry: OutlineEntry, first_line: int) -> str: + parts = [entry.title, f"level {entry.level}"] + if entry.page is not None: + parts.append(f"page {entry.page}") + if entry.line is not None: + parts.append(f"line {first_line + entry.line - 1}") + return " | ".join(parts) + + def companion_name(md_name: str) -> str: """`derived.md` -> `derived.sheets`: the folder beside a `.md` for its `files`.""" stem = md_name[:-3] if md_name.lower().endswith(".md") else md_name diff --git a/tools/chemenu/filters/pdf.py b/tools/chemenu/filters/pdf.py index c716d0a..8fe93cc 100644 --- a/tools/chemenu/filters/pdf.py +++ b/tools/chemenu/filters/pdf.py @@ -1,10 +1,13 @@ -"""PDF, read with `pypdf` (imported when a PDF is derived, never at start-up).""" +"""PDF: the text and the outline are read with `pypdf`, the look of the characters (highlight, +colour, strike-through, images) with `pdfminer.six` in `pdf_layout`. Both are imported when a PDF is +derived, never at start-up.""" from __future__ import annotations import io +from typing import Optional from chemenu.errors import ValidationError -from chemenu.filters import Derivation, Filter, register +from chemenu.filters import Derivation, Filter, OutlineEntry, pdf_layout, register SCAN_CHARS_PER_PAGE = 50 @@ -12,7 +15,7 @@ SCAN_CHARS_PER_PAGE = 50 class PdfFilter(Filter): name = "pdf" extensions = (".pdf",) - requires = ("pypdf",) + requires = ("pypdf", "pdfminer") def derive(self, data: bytes, name: str) -> Derivation: import pypdf @@ -36,13 +39,51 @@ class PdfFilter(Filter): if not pages: raise ValidationError(f"{name} has no pages.") - blocks = [f"\n{text}".rstrip() for number, text in enumerate(pages, 1)] characters = sum(len(text) for text in pages) warnings: list[str] = [] checks = [ "Page breaks are the markers; tables and multi-column layouts may " "come out in reading order, not as laid out - check figures against the original." ] + fields: list[tuple[str, "str | list[str]"]] = [("pages", str(len(pages)))] + + layouts: list[pdf_layout.PageLayout] = [] + try: + layouts = pdf_layout.analyse(data) + except Exception as exc: # pdfminer.six raises anything on a document it cannot parse + warnings.append( + f"The page layout could not be read ({type(exc).__name__}: {exc}); the text has no " + "markings and images are not reported." + ) + if layouts and len(layouts) != len(pages): + warnings.append( + f"The layout has {len(layouts)} page(s), the text {len(pages)}; the text has no " + "markings and images are not reported." + ) + layouts = [] + if layouts: + annotation = pdf_layout.annotate(pages, layouts) + pages = list(annotation.texts) + warnings.extend(annotation.warnings) + if annotation.markings: + fields.append(("markings", list(annotation.markings))) + image_pages = [number for number, layout in enumerate(layouts, 1) if layout.images] + if image_pages: + checks.append( + f"Images on page(s) {pdf_layout.page_ranges(image_pages)} are not contained in the text - " + "check them against the original." + ) + + blocks = [f"\n{text}".rstrip() for number, text in enumerate(pages, 1)] + marker_lines, line = {}, 1 + for number, block in enumerate(blocks, 1): + marker_lines[number] = line + line += block.count("\n") + 2 + outline: list[OutlineEntry] = [] + try: + outline = _outline(reader, len(pages), marker_lines) + except Exception as exc: # pypdf raises anything on a damaged outline tree + warnings.append(f"The outline could not be read ({type(exc).__name__}: {exc}); the header has none.") if characters < SCAN_CHARS_PER_PAGE * len(pages): warnings.append( f"The PDF yields {characters} character(s) over {len(pages)} page(s): it is probably " @@ -51,10 +92,33 @@ class PdfFilter(Filter): return Derivation( text="\n\n".join(blocks), title=title, - fields=(("pages", str(len(pages))),), + fields=tuple(fields), warnings=tuple(warnings), checks=tuple(checks), + outline=tuple(outline), ) +def _outline(reader, page_count: int, marker_lines: dict[int, int]) -> list[OutlineEntry]: + entries: list[OutlineEntry] = [] + + def walk(items, level: int) -> None: + for item in items: + if isinstance(item, list): + walk(item, level + 1) + continue + title = " ".join(str(getattr(item, "title", "") or "").split()) + if not title: + continue + try: + index = reader.get_destination_page_number(item) + except Exception: # a destination that does not resolve is kept, without a page + index = None + page: Optional[int] = index + 1 if isinstance(index, int) and 0 <= index < page_count else None + entries.append(OutlineEntry(title, level, page, marker_lines.get(page) if page else None)) + + walk(reader.outline, 1) + return entries + + register(PdfFilter()) diff --git a/tools/chemenu/filters/pdf_layout.py b/tools/chemenu/filters/pdf_layout.py new file mode 100644 index 0000000..3390f3b --- /dev/null +++ b/tools/chemenu/filters/pdf_layout.py @@ -0,0 +1,318 @@ +"""What the PDF filter reads from the page layout: per character its text colour, the colour +filled behind it, a line through it and a link over it - and the mapping of those onto the text +`pypdf` extracted. Read with `pdfminer.six` (imported when `analyse` runs, never at start-up). + +`pypdf` stays the source of the text, so a PDF's wording does not change; `pdfminer.six` only +says what each character looks like. The two are matched page by page over their non-space +characters, and the markings are laid onto the `pypdf` text. The markings follow the visible +form and nothing else: the filter has no model, and what a colour means is written in the +document itself.""" +from __future__ import annotations + +import difflib +import io +import itertools +import unicodedata +from dataclasses import dataclass +from typing import Iterable, Optional + +MIN_COVERAGE = 0.9 # a page whose text matches the layout less than this gets no markings +MAX_HIGHLIGHT_LINES = 2.5 # a fill taller than this many text heights is a panel, not a highlight +STRIKE_BAND = (0.3, 0.75) # where in the character's height a line counts as going through it +FLAT = 1.5 # a filled shape this thin (points) is a line + +NAMES = { + "#ffff00": "gelb", "#00ff00": "grün", "#ff0000": "rot", "#0000ff": "blau", + "#00ffff": "türkis", "#ff00ff": "pink", "#ffa500": "orange", "#ffc000": "orange", + "#ff9900": "orange", "#808080": "grau", "#c0c0c0": "hellgrau", "#d3d3d3": "hellgrau", + "#000000": "schwarz", "#000080": "dunkelblau", "#008080": "blaugrün", "#008000": "dunkelgrün", + "#800080": "violett", "#800000": "dunkelrot", "#808000": "olivgelb", +} + +Flags = tuple # (background hex | None, True | None for struck, colour hex | None) +LAYERS = 3 + + +@dataclass(frozen=True) +class Glyph: + text: str + colour: Optional[str] + background: Optional[str] + struck: bool + linked: bool + + +@dataclass(frozen=True) +class PageLayout: + glyphs: tuple[Glyph, ...] + images: bool + + +@dataclass(frozen=True) +class Annotation: + texts: tuple[str, ...] + markings: tuple[str, ...] # the legend + warnings: tuple[str, ...] + + +def colour_name(value: str) -> str: + return NAMES.get(value, value) + + +def page_ranges(pages: Iterable[int]) -> str: + """`[6, 7, 8, 10]` -> `6-8, 10`.""" + out = [] + for _, group in itertools.groupby(enumerate(sorted(set(pages))), key=lambda pair: pair[1] - pair[0]): + run = [number for _, number in group] + out.append(str(run[0]) if len(run) == 1 else f"{run[0]}-{run[-1]}") + return ", ".join(out) + + +def _hex(value) -> Optional[str]: + """A fill colour as `#rrggbb`: a gray level, RGB or CMYK. None for anything else (a pattern).""" + if isinstance(value, (int, float)): + value = (value,) + if not isinstance(value, (tuple, list)) or not all(isinstance(v, (int, float)) for v in value): + return None + values = [min(1.0, max(0.0, float(v))) for v in value] + if len(values) == 1: + red = green = blue = values[0] + elif len(values) == 3: + red, green, blue = values + elif len(values) == 4: + cyan, magenta, yellow, black = values + red, green, blue = ((1 - c) * (1 - black) for c in (cyan, magenta, yellow)) + else: + return None + return "#%02x%02x%02x" % (round(red * 255), round(green * 255), round(blue * 255)) + + +def _is_white(value: str) -> bool: + return all(int(value[i:i + 2], 16) >= 0xF8 for i in (1, 3, 5)) + + +def analyse(data: bytes) -> list[PageLayout]: + """The glyphs and images of each page, in page order. Raises whatever `pdfminer.six` + raises on a document it cannot read.""" + from pdfminer.converter import PDFPageAggregator + from pdfminer.layout import LTChar, LTCurve, LTFigure, LTImage, LTLine + from pdfminer.pdfinterp import PDFPageInterpreter, PDFResourceManager + from pdfminer.pdfpage import PDFPage + from pdfminer.pdftypes import resolve1 + + def flatten(container): + for item in container: + if isinstance(item, LTFigure): + yield from flatten(item) + else: + yield item + + def link_boxes(page) -> Optional[list[tuple[float, float, float, float]]]: + """Link rectangles in the coordinates of the layout; None on a rotated page, where + they cannot be told apart from the layout's.""" + if page.rotate % 360: + return None + origin_x, origin_y = page.mediabox[0], page.mediabox[1] + boxes = [] + for annotation in resolve1(page.annots) or []: + annotation = resolve1(annotation) + subtype = resolve1(annotation.get("Subtype")) if hasattr(annotation, "get") else None + if getattr(subtype, "name", None) != "Link": + continue + rect = [resolve1(v) for v in resolve1(annotation.get("Rect")) or []] + if len(rect) == 4 and all(isinstance(v, (int, float)) for v in rect): + boxes.append(( + min(rect[0], rect[2]) - origin_x, min(rect[1], rect[3]) - origin_y, + max(rect[0], rect[2]) - origin_x, max(rect[1], rect[3]) - origin_y, + )) + return boxes + + manager = PDFResourceManager() + device = PDFPageAggregator(manager, laparams=None) + interpreter = PDFPageInterpreter(manager, device) + layouts = [] + for page in PDFPage.get_pages(io.BytesIO(data)): + interpreter.process_page(page) + links = link_boxes(page) + chars, highlights, lines, images = [], [], [], False + for order, item in enumerate(flatten(device.get_result())): + if isinstance(item, LTChar): + chars.append((order, item)) + elif isinstance(item, LTImage): + images = True + elif isinstance(item, LTCurve): + x0, y0, x1, y1 = item.bbox + fill = _hex(item.non_stroking_color) if item.fill else None + if y1 - y0 <= FLAT and x1 - x0 >= 1 and (fill is not None or item.stroke or isinstance(item, LTLine)): + lines.append((x0, x1, (y0 + y1) / 2)) + elif fill is not None and not _is_white(fill): + highlights.append((order, item.bbox, fill)) + glyphs = [] + for order, char in chars: + if char.get_text().isspace(): + continue + x0, y0, x1, y1 = char.bbox + centre_x, centre_y, height = (x0 + x1) / 2, (y0 + y1) / 2, y1 - y0 + background = next(( + colour for placed, (bx0, by0, bx1, by1), colour in reversed(highlights) + if placed < order and bx0 <= centre_x <= bx1 and by0 <= centre_y <= by1 + and by1 - by0 <= MAX_HIGHLIGHT_LINES * height + ), None) + struck = any( + lx0 <= centre_x <= lx1 and y0 + STRIKE_BAND[0] * height <= ly <= y0 + STRIKE_BAND[1] * height + for lx0, lx1, ly in lines + ) + linked = links is not None and any( + bx0 <= centre_x <= bx1 and by0 <= centre_y <= by1 for bx0, by0, bx1, by1 in links + ) + glyphs.append(Glyph( + char.get_text(), _hex(char.graphicstate.ncolor), background, struck, + linked or links is None, + )) + layouts.append(PageLayout(tuple(glyphs), images)) + return layouts + + +def body_colour(layouts: Iterable[PageLayout]) -> Optional[str]: + """The fill colour with the most characters: the colour of the running text.""" + counts: dict[str, int] = {} + for layout in layouts: + for glyph in layout.glyphs: + if glyph.colour is not None: + counts[glyph.colour] = counts.get(glyph.colour, 0) + 1 + return max(counts, key=lambda colour: (counts[colour], colour)) if counts else None + + +def _flags(glyph: Glyph, body: Optional[str]) -> Flags: + other = glyph.colour if glyph.colour is not None and glyph.colour != body and not glyph.linked else None + return (glyph.background, True if glyph.struck else None, other) + + +def _align(text: str, glyphs: tuple[Glyph, ...], flags: list[Flags]) -> tuple[dict[int, Flags], float]: + """The flags of each character of `text` that has a counterpart among the glyphs, and the + share of `text`'s non-space characters that has one.""" + layout_chars, layout_flags = [], [] + for glyph, flag in zip(glyphs, flags): + for char in unicodedata.normalize("NFKC", glyph.text): + if not char.isspace(): + layout_chars.append(char) + layout_flags.append(flag) + text_chars, text_index = [], [] + for index, original in enumerate(text): + for char in unicodedata.normalize("NFKC", original): + if not char.isspace(): + text_chars.append(char) + text_index.append(index) + found: dict[int, Flags] = {} + matched = 0 + + def take(layout_at: int, text_at: int, size: int) -> None: + nonlocal matched + for offset in range(size): + matched += 1 + found.setdefault(text_index[text_at + offset], layout_flags[layout_at + offset]) + + # the two orders usually agree: only what lies between the common ends goes through difflib + head = 0 + while head < min(len(layout_chars), len(text_chars)) and layout_chars[head] == text_chars[head]: + head += 1 + tail = 0 + while ( + tail < min(len(layout_chars), len(text_chars)) - head + and layout_chars[-1 - tail] == text_chars[-1 - tail] + ): + tail += 1 + take(0, 0, head) + take(len(layout_chars) - tail, len(text_chars) - tail, tail) + middle_layout, middle_text = layout_chars[head:len(layout_chars) - tail], text_chars[head:len(text_chars) - tail] + if middle_layout and middle_text: + matcher = difflib.SequenceMatcher(None, "".join(middle_layout), "".join(middle_text), autojunk=False) + for block in matcher.get_matching_blocks(): + take(head + block.a, head + block.b, block.size) + return found, (matched / len(text_chars) if text_chars else 1.0) + + +def _line_flags(line: str, found: dict[int, Flags], start: int) -> list[Optional[Flags]]: + """Flags per character of one line. A space between two characters that agree on a layer + takes that layer's value, so a marking runs across the gap and never starts or ends on a space.""" + flags: list[Optional[Flags]] = [None if char.isspace() else found.get(start + j) for j, char in enumerate(line)] + solid = [j for j, char in enumerate(line) if not char.isspace()] + for before, after in zip(solid, solid[1:]): + if after - before > 1 and flags[before] is not None and flags[after] is not None: + shared = tuple(a if a == b else None for a, b in zip(flags[before], flags[after])) + if any(value is not None for value in shared): + for j in range(before + 1, after): + flags[j] = shared + return flags + + +def _open_close(layer: int, value) -> tuple[str, str]: + if layer == 0: + return f'', "" + return ("~~", "~~") if layer == 1 else ("*", "*") + + +def _emit(items: list[tuple[str, Optional[Flags]]], layer: int, record) -> str: + if layer == LAYERS: + return "".join(char for char, _ in items) + out = [] + for value, group in itertools.groupby(items, key=lambda item: item[1][layer] if item[1] else None): + inner = _emit(list(group), layer + 1, record) + if value is None: + out.append(inner) + else: + opening, closing = _open_close(layer, value) + out.append(opening + inner + closing) + record(layer, value) + return "".join(out) + + +def annotate(texts: list[str], layouts: list[PageLayout]) -> Annotation: + """`texts` with the markings laid on, the legend and the warnings. A page without + markings comes back as it is.""" + body = body_colour(layouts) + result = list(texts) + places: dict[tuple[int, object], list[int]] = {} + unmatched: list[int] = [] + for number, (text, layout) in enumerate(zip(texts, layouts), 1): + flags = [_flags(glyph, body) for glyph in layout.glyphs] + if not any(any(value is not None for value in flag) for flag in flags): + continue + found, coverage = _align(text, layout.glyphs, flags) + if coverage < MIN_COVERAGE: + unmatched.append(number) + continue + + def record(layer: int, value, page: int = number) -> None: + places.setdefault((layer, value if layer != 1 else True), []).append(page) + + lines, start = [], 0 + for line in text.split("\n"): + line_flags = _line_flags(line, found, start) + start += len(line) + 1 + lines.append(line if not any(line_flags) else _emit(list(zip(line, line_flags)), 0, record)) + result[number - 1] = "\n".join(lines) + + markings = [] + for (layer, value), pages in sorted(places.items(), key=lambda entry: (entry[0][0], str(entry[0][1]))): + if layer == 0: + opening, closing = _open_close(0, value) + markings.append(f"{opening}…{closing} | background {value} ({colour_name(value)}) | places {len(pages)} | pages {page_ranges(pages)}") + strike = places.get((1, True)) + if strike: + markings.append(f"~~…~~ | strike-through | places {len(strike)} | pages {page_ranges(strike)}") + colours = {value: pages for (layer, value), pages in places.items() if layer == 2} + if colours: + all_pages = [page for pages in colours.values() for page in pages] + names = ", ".join(sorted(colours)) + markings.append( + f"*…* | text colour other than the running text ({body}), not under a link: {names} | " + f"places {len(all_pages)} | pages {page_ranges(all_pages)}" + ) + warnings = [] + if unmatched: + warnings.append( + f"Page(s) {page_ranges(unmatched)}: under {round(MIN_COVERAGE * 100)}% of the text could be matched to the " + "page layout, so the markings (highlight, colour, strike-through) are left out there." + ) + return Annotation(tuple(result), tuple(markings), tuple(warnings)) diff --git a/tools/chemenu/tests/binary_formats.py b/tools/chemenu/tests/binary_formats.py index d205884..aba3a27 100644 --- a/tools/chemenu/tests/binary_formats.py +++ b/tools/chemenu/tests/binary_formats.py @@ -5,7 +5,8 @@ valid document is settled by the package that reads it in the tests.""" from __future__ import annotations import struct -from typing import Optional +from dataclasses import dataclass, field +from typing import Optional, Union END, FREE, FATSECT = 0xFFFFFFFE, 0xFFFFFFFF, 0xFFFFFFFD @@ -50,6 +51,137 @@ def build_pdf(pages: list[list[str]], *, title: Optional[str] = None) -> bytes: return bytes(out) +Run = Union[str, tuple[str, dict]] # text, or text and options: color, highlight, strike, link + + +@dataclass +class StyledPage: + """Lines of runs. A run's options: `color` (a gray level, an RGB or a CMYK tuple), + `highlight` (an RGB filled behind the text), `strike` (a line through it), `link` + (a /Link annotation over it).""" + + lines: list[list[Run]] = field(default_factory=list) + image: bool = False + + +CHAR_WIDTH, LINE_HEIGHT, LEFT, TOP = 6, 14, 72, 720 # Courier 10 pt + + +def _operator(color: "float | tuple[float, ...]", fill: bool) -> str: + values = (color,) if isinstance(color, (int, float)) else tuple(color) + name = {1: "g", 3: "rg", 4: "k"}[len(values)] + return " ".join(f"{v:g}" for v in values) + " " + (name if fill else name.upper()) + + +def build_styled_pdf( + pages: list[StyledPage], + *, + outline: "list[tuple[str, int, Optional[int]]]" = (), + title: Optional[str] = None, +) -> bytes: + """A PDF in Courier 10 pt where each run can be coloured, highlighted, struck through + or linked, a page can carry an image, and `outline` is a list of `(title, level, + page index from 0 or None for an entry that goes nowhere)`. Character cells are 6 pt wide, so the geometry is exact.""" + objects: dict[int, bytes] = {} + next_id = [3] + + def allocate() -> int: + number = next_id[0] + next_id[0] += 1 + return number + + page_ids = [allocate() for _ in pages] + font_id, image_id = allocate(), allocate() + for number, page in zip(page_ids, pages): + ops, annots = ["BT /F1 10 Tf ET"], [] + for row, line in enumerate(page.lines): + y, x = TOP - LINE_HEIGHT * row, LEFT + for run in line: + text, options = (run, {}) if isinstance(run, str) else run + width = CHAR_WIDTH * len(text) + if "highlight" in options: + ops.append(f"{_operator(options['highlight'], True)} {x} {y - 3} {width} 12 re f 0 g") + if "color" in options: + ops.append(_operator(options["color"], True)) + escaped = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + ops.append(f"BT /F1 10 Tf {x} {y} Td ({escaped}) Tj ET") + if "color" in options: + ops.append("0 g") + if options.get("strike"): + ops.append(f"0 G 0.7 w {x} {y + 3.2} m {x + width} {y + 3.2} l S") + if options.get("link"): + annots.append( + f"<< /Type /Annot /Subtype /Link /Rect [{x} {y - 3} {x + width} {y + 9}] /Border [0 0 0] " + "/A << /S /URI /URI (https://example.org/) >> >>" + ) + x += width + if page.image: + ops.append("q 40 0 0 40 72 400 cm /Im1 Do Q") + stream = "\n".join(ops).encode("latin-1") + content_id = allocate() + objects[content_id] = b"<< /Length %d >>\nstream\n" % len(stream) + stream + b"\nendstream" + xobject = f"/XObject << /Im1 {image_id} 0 R >>" if page.image else "" + objects[number] = ( + f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents {content_id} 0 R " + f"/Resources << /Font << /F1 {font_id} 0 R >> {xobject} >> /Annots [{' '.join(annots)}] >>" + ).encode() + objects[font_id] = b"<< /Type /Font /Subtype /Type1 /BaseFont /Courier >>" + objects[image_id] = b"<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 /Length 1 >>\nstream\n\x80\nendstream" + objects[1] = b"<< /Type /Catalog /Pages 2 0 R >>" + objects[2] = f"<< /Type /Pages /Kids [{' '.join(f'{n} 0 R' for n in page_ids)}] /Count {len(pages)} >>".encode() + if outline: + root = allocate() + ids = [allocate() for _ in outline] + parent_of: list[Optional[int]] = [] + stack: list[tuple[int, int]] = [] # (level, index) + for index, (_title, level, _page) in enumerate(outline): + while stack and stack[-1][0] >= level: + stack.pop() + parent_of.append(stack[-1][1] if stack else None) + stack.append((level, index)) + for index, (entry_title, _level, page_index) in enumerate(outline): + siblings = [i for i, parent in enumerate(parent_of) if parent == parent_of[index]] + children = [i for i, parent in enumerate(parent_of) if parent == index] + position = siblings.index(index) + parts = [ + f"/Title ({entry_title})", + f"/Parent {(ids[parent_of[index]] if parent_of[index] is not None else root)} 0 R", + ] + if page_index is not None: + parts.append(f"/Dest [{page_ids[page_index]} 0 R /Fit]") + if position > 0: + parts.append(f"/Prev {ids[siblings[position - 1]]} 0 R") + if position < len(siblings) - 1: + parts.append(f"/Next {ids[siblings[position + 1]]} 0 R") + if children: + parts += [f"/First {ids[children[0]]} 0 R", f"/Last {ids[children[-1]]} 0 R", f"/Count {len(children)}"] + objects[ids[index]] = ("<< " + " ".join(parts) + " >>").encode("latin-1") + top = [i for i, parent in enumerate(parent_of) if parent is None] + objects[root] = f"<< /Type /Outlines /First {ids[top[0]]} 0 R /Last {ids[top[-1]]} 0 R /Count {len(top)} >>".encode() + objects[1] = f"<< /Type /Catalog /Pages 2 0 R /Outlines {root} 0 R >>".encode() + trailer = "/Root 1 0 R" + if title is not None: + info_id = allocate() + objects[info_id] = f"<< /Title ({title}) >>".encode("latin-1") + trailer += f" /Info {info_id} 0 R" + return _serialize(objects, trailer) + + +def _serialize(objects: dict[int, bytes], trailer: str) -> bytes: + out = bytearray(b"%PDF-1.4\n") + offsets = {} + for number in sorted(objects): + offsets[number] = len(out) + out += f"{number} 0 obj\n".encode() + objects[number] + b"\nendobj\n" + xref = len(out) + size = max(objects) + 1 + out += f"xref\n0 {size}\n".encode() + b"0000000000 65535 f \n" + for number in range(1, size): + out += f"{offsets[number]:010d} 00000 n \n".encode() + out += f"trailer\n<< /Size {size} {trailer} >>\nstartxref\n{xref}\n%%EOF\n".encode() + return bytes(out) + + class _Node: def __init__(self, name: str, kind: int, data: bytes = b"") -> None: self.name, self.kind, self.data = name, kind, data diff --git a/tools/chemenu/tests/test_raw_derive_binary.py b/tools/chemenu/tests/test_raw_derive_binary.py index 63a1c1f..6537484 100644 --- a/tools/chemenu/tests/test_raw_derive_binary.py +++ b/tools/chemenu/tests/test_raw_derive_binary.py @@ -1,5 +1,5 @@ -"""`wikitool raw derive` for the two formats read through a package: PDF (`pypdf`) and -Outlook `.msg` (`olefile`). The documents come from `binary_formats`, so no binary sits +"""`wikitool raw derive` for the two formats read through a package: PDF (`pypdf` for the text and +the outline, `pdfminer.six` for the look of the characters) and Outlook `.msg` (`olefile`). The documents come from `binary_formats`, so no binary sits in the repository; whether the builders write valid files is settled by the packages reading them here. The tests that need a package skip where it is not installed - CI installs `tools/requirements.txt`, so they run there.""" @@ -7,13 +7,15 @@ from __future__ import annotations import importlib.util import io +import re from pathlib import Path import pytest from chemenu import filters from chemenu.commands import doctor -from chemenu.tests.binary_formats import build_msg, build_pdf +from chemenu.filters import pdf_layout +from chemenu.tests.binary_formats import StyledPage, build_msg, build_pdf, build_styled_pdf from chemenu.tests.test_raw_derive import ( # noqa: F401 (tree is a fixture) _derive, _header_and_text, @@ -27,6 +29,7 @@ from chemenu.tests.test_raw_derive import ( # noqa: F401 (tree is a fixture) ) needs_pypdf = pytest.mark.skipif(importlib.util.find_spec("pypdf") is None, reason="pypdf is not installed") +needs_pdfminer = pytest.mark.skipif(importlib.util.find_spec("pdfminer") is None, reason="pdfminer.six is not installed") needs_olefile = pytest.mark.skipif(importlib.util.find_spec("olefile") is None, reason="olefile is not installed") LONG_LINE = "A sentence long enough that the page is clearly not a scan of a picture." @@ -125,6 +128,258 @@ def test_a_package_missing_inside_a_zip_refuses_the_whole_zip(tree, monkeypatch) assert _snapshot(tree / "incoming") == before +# --- PDF: outline, markings, images -------------------------------------------- + +YELLOW, GREEN, RED, BLUE = (1, 1, 0), (0, 1, 0), (1, 0, 0), (0, 0, 1) + + +def _derive_pdf(tree, pdf: bytes, name: str = "doc.pdf"): + _put(tree, name, pdf) + _derive(f"incoming/{name}") + return _header_and_text(tree / "incoming" / name[:-4] / "derived.md") + + +def _markings_stripped(text: str) -> str: + return re.sub(r'||~~|\*', "", text) + + +@needs_pypdf +@needs_pdfminer +def test_the_outline_goes_into_the_header_with_level_page_and_line(tree): + pdf = build_styled_pdf( + [StyledPage([["One"]]), StyledPage([["Two", ], ["more"]]), StyledPage([["Three"]])], + outline=[("Part A", 1, 0), ("Rule 16 (hands)", 2, 1), ("Signals", 3, 1), ("Part B", 1, 2), ("Lost entry", 2, None)], + ) + _put(tree, "doc.pdf", pdf) + _derive("incoming/doc.pdf") + path = tree / "incoming/doc/derived.md" + header, _text = _header_and_text(path) + file_lines = path.read_text(encoding="utf-8").split("\n") + + assert header["outline"] == [ + "Part A | level 1 | page 1 | line " + str(file_lines.index("") + 1), + "Rule 16 (hands) | level 2 | page 2 | line " + str(file_lines.index("") + 1), + "Signals | level 3 | page 2 | line " + str(file_lines.index("") + 1), + "Part B | level 1 | page 3 | line " + str(file_lines.index("") + 1), + "Lost entry | level 2", + ] + for entry in header["outline"][:4]: + _title, _level, page, line = entry.rsplit(" | ", 3) + assert file_lines[int(line.split()[1]) - 1] == f"" + + +@needs_pypdf +@needs_pdfminer +def test_a_pdf_without_outline_or_markings_has_neither_field_and_no_image_check(tree, capsys): + header, text = _derive_pdf(tree, build_styled_pdf([StyledPage([[LONG_LINE]])], title="T")) + out = capsys.readouterr().out + assert "outline" not in header and "markings" not in header + assert "old rule and new rule end' in text + assert 'second yellow' in text + legend = {entry.split(" | ")[0]: entry for entry in header["markings"]} + assert "places 2 | pages 1-2" in legend['…'] + assert "#ffff00" in legend['…'] + assert "places 1 | pages 1" in legend['…'] + + +@needs_pypdf +@needs_pdfminer +def test_a_background_without_a_name_is_marked_with_its_hex_value(tree): + pdf = build_styled_pdf([StyledPage([[("odd", {"highlight": (0.2, 0.4, 0.6)})]])]) + _header, text = _derive_pdf(tree, pdf) + assert 'odd' in text + + +@needs_pypdf +@needs_pdfminer +def test_a_white_fill_is_not_a_highlight(tree): + pdf = build_styled_pdf([StyledPage([[("on white", {"highlight": (1, 1, 1)})]])]) + header, text = _derive_pdf(tree, pdf) + assert "~~*two words*~~' in text + + +@needs_pypdf +@needs_pdfminer +def test_a_marking_never_runs_over_a_line_or_a_page_marker(tree): + pdf = build_styled_pdf([ + StyledPage([[("first line", {"highlight": YELLOW})], [("second line", {"highlight": YELLOW})]]), + StyledPage([[("third page", {"highlight": YELLOW, "strike": True})]]), + ]) + _header, text = _derive_pdf(tree, pdf) + for line in text.splitlines(): + assert line.count("") and line.count("~~") % 2 == 0, line + assert 'first line\nsecond line' in text + assert "\n\n\nhighlighted' not in text + assert 'fine' in text + assert "WARN" in out and "Page(s) 1" in out and "left out" in out + + +@needs_pypdf +@needs_pdfminer +def test_a_layout_that_cannot_be_read_costs_the_markings_not_the_derivation(monkeypatch, tree, capsys): + def broken(data): + raise ValueError("boom") + + monkeypatch.setattr(pdf_layout, "analyse", broken) + header, text = _derive_pdf(tree, build_styled_pdf([StyledPage([[("kept", {"highlight": YELLOW})]])], outline=[("Top", 1, 0)])) + out = capsys.readouterr().out + assert "kept" in text and "" + + +def test_characters_are_matched_across_a_difference_in_the_middle(): + flag = ("#ffff00", None, None) + glyphs = tuple(pdf_layout.Glyph(c, "#000000", "#ffff00" if i >= 4 else None, False, False) for i, c in enumerate("start ghost middle end")) + found, coverage = pdf_layout._align("start middle end", glyphs, [pdf_layout._flags(g, "#000000") for g in glyphs]) + assert coverage == 1.0 + assert [found.get(i) for i in (0, 6, 7, 13)] == [(None, None, None), flag, flag, flag] + + +def test_page_ranges_join_neighbours(): + assert pdf_layout.page_ranges([6, 7, 8, 10, 12, 13, 8]) == "6-8, 10, 12-13" + assert pdf_layout.page_ranges([]) == "" + + +def test_colours_are_read_from_gray_rgb_and_cmyk(): + assert pdf_layout._hex(0.5) == "#808080" + assert pdf_layout._hex((1.0, 0.0, 0.0)) == "#ff0000" + assert pdf_layout._hex((0.0, 0.0, 1.0, 0.0)) == "#ffff00" + assert pdf_layout._hex(("pattern",)) is None and pdf_layout._hex(None) is None + + # --- Outlook .msg -------------------------------------------------------------- diff --git a/tools/requirements.txt b/tools/requirements.txt index 37c08ab..1740e8d 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -5,4 +5,5 @@ jsonschema>=4.18 referencing>=0.30 icalendar>=6.0 pypdf>=5.0 +pdfminer.six>=20231228 olefile>=0.47