From 0e6a50aa985dcbb17952f5d92212fde76f26602f Mon Sep 17 00:00:00 2001 From: Torben Nehmer Date: Thu, 8 Oct 2026 14:02:16 +0200 Subject: [PATCH] feat(raw): raw derive reads PDF and Outlook MSG; doctor shows each import filter PDF is read with pypdf (page markers, scan warning, encrypted files refused), Outlook .msg with olefile as a container like an .eml, and a .msg attached to a mail is quoted. Both packages are in tools/requirements.txt and imported only when such a file is derived, so a missing one stops that format alone. doctor reports one import-filter line per package-backed format. Files changed: - CHANGES.md - VERSION - docs/how-to/ingest-a-source.md - docs/reference/commands/instance-health.md - docs/reference/commands/raw-material-and-uploads.md - tools/CONTRACT.md - tools/README.md - tools/chemenu/commands/doctor.py - tools/chemenu/commands/raw_cmd.py - tools/chemenu/filters/__init__.py - tools/chemenu/filters/mail.py - tools/chemenu/filters/msg.py - tools/chemenu/filters/pdf.py - tools/chemenu/tests/binary_formats.py - tools/chemenu/tests/test_raw_derive_binary.py - tools/requirements.txt Co-Authored-By: Claude Code Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2 --- CHANGES.md | 3 +- VERSION | 2 +- docs/how-to/ingest-a-source.md | 6 +- docs/reference/commands/instance-health.md | 1 + .../commands/raw-material-and-uploads.md | 2 +- tools/CONTRACT.md | 3 +- tools/README.md | 6 +- tools/chemenu/commands/doctor.py | 26 +- tools/chemenu/commands/raw_cmd.py | 8 +- tools/chemenu/filters/__init__.py | 2 +- tools/chemenu/filters/mail.py | 25 +- tools/chemenu/filters/msg.py | 213 +++++++++++++++ tools/chemenu/filters/pdf.py | 60 +++++ tools/chemenu/tests/binary_formats.py | 224 ++++++++++++++++ tools/chemenu/tests/test_raw_derive_binary.py | 246 ++++++++++++++++++ tools/requirements.txt | 2 + 16 files changed, 812 insertions(+), 17 deletions(-) create mode 100644 tools/chemenu/filters/msg.py create mode 100644 tools/chemenu/filters/pdf.py create mode 100644 tools/chemenu/tests/binary_formats.py create mode 100644 tools/chemenu/tests/test_raw_derive_binary.py diff --git a/CHANGES.md b/CHANGES.md index f2b8524..67da680 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,7 +14,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. --- -## 9.0.0-beta.15 - 2026-10-08 - raw derive unpacks ZIP archives and mails with attachments into a folder +## 9.0.0-beta.16 - 2026-10-08 - raw derive reads PDF and Outlook MSG through lazily imported packages; doctor shows each import filter **Author:** Torben Nehmer @@ -37,6 +37,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. - Asset-Preflight: Verschieben aus dem Entpackordner wird bis ca. 30 s wiederholt, nennt die Systemmeldung und rollt nach endgültigem Scheitern zurück - raw derive reads PDF-free documents: Office files and mail become a folder with the original and a derived text - raw derive unpacks ZIP archives and mails with attachments into a folder +- raw derive reads PDF and Outlook MSG through lazily imported packages; doctor shows each import filter **Low impact** - version bump: Abschlussmeldung nennt Stichpunkt oder Themenabsatz statt pauschal Prosa diff --git a/VERSION b/VERSION index 74ad31d..f0560d2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -9.0.0-beta.15 +9.0.0-beta.16 diff --git a/docs/how-to/ingest-a-source.md b/docs/how-to/ingest-a-source.md index 13af1a1..fbbd718 100644 --- a/docs/how-to/ingest-a-source.md +++ b/docs/how-to/ingest-a-source.md @@ -16,11 +16,13 @@ pick a directory: - **A web page** needs no download of your own. Name the URL to the agent, and `tools/wikitool raw fetch` puts two files into `incoming/`: the HTML exactly as received, and a text derived from it with a header recording where and when it was fetched. -- **A document that is not text** - a Word, PowerPoint or Excel file, a mail or a ZIP archive - is +- **A document that is not text** - a PDF, a Word, PowerPoint or Excel file, a mail (`.eml` or + Outlook `.msg`) or a ZIP archive - is put into `incoming/` as it is. `tools/wikitool raw derive incoming/` turns it into a folder with the untouched original and a readable text beside it, with a header saying how it was made. The folder is accepted whole; the agent runs this itself when it meets such a file. A ZIP is - unpacked into a folder instead, and a mail's attachments land beside it. + unpacked into a folder instead, and a mail's attachments land beside it. A scanned PDF has no + text to derive - the tool warns, and there is no OCR. - **A page behind a paywall or a login** is saved from your browser into `incoming/`, HTML only. The agent derives the same text from that file with `raw fetch --html`. diff --git a/docs/reference/commands/instance-health.md b/docs/reference/commands/instance-health.md index 574b1e3..fee6830 100644 --- a/docs/reference/commands/instance-health.md +++ b/docs/reference/commands/instance-health.md @@ -28,6 +28,7 @@ tools/wikitool doctor --json **Behaviour** - Checks dependencies (Python, ripgrep), author resolution, stack version, git identity/branch/remote, published skills, the kb/raw/reports/work/instructions structure, and generated files. +- Import filters (`import-filter-`): one line for each `raw derive` format that reads through a package (PDF, Outlook `.msg`) - a `WARN` naming the package when it is missing, since that costs only its own format; fixed by running `tools/preflight.sh` again. - Tool paths (`tool-paths`): `.wikitool-tools.json` written by a preflight that finished, every tool `tools/prerequisites.txt` names for this platform recorded, and every recorded path still there - a `FAIL` otherwise, fixed by running `tools/preflight.sh` again (PowerShell 7: `tools/preflight.ps1`). - Install folder (`install-dir`): on Windows with long paths off, a `FAIL` when the folder holding `tools/` is longer than `tools/prerequisites.txt` allows (95 characters) - the limit the preflight enforces before it sets anything up. - PowerShell (`execution-policy`, `script-marks`; Windows only, `OK` elsewhere): a `FAIL` when the effective execution policy is `Restricted` or `AllSigned` - the line then says whether a group policy sets it, which only whoever administers the computer can change - and a `FAIL` when a script under `tools/` carries a Mark of the Web from the internet zone, which a browser download unpacked in Explorer leaves behind and `Invoke-WebRequest` plus `tar` do not. `tools/preflight.ps1` checks the same two things before it stops. diff --git a/docs/reference/commands/raw-material-and-uploads.md b/docs/reference/commands/raw-material-and-uploads.md index f98e6da..f7391ca 100644 --- a/docs/reference/commands/raw-material-and-uploads.md +++ b/docs/reference/commands/raw-material-and-uploads.md @@ -118,7 +118,7 @@ tools/wikitool raw derive incoming/scans.zip - A single file directly in `incoming/` becomes a folder named after its stem (or `--name`) holding `original.` - the original bytes, moved - and `derived.md`. The fixed names keep the path inside the path budget; the header of `derived.md` records the file's own name as `original_name`. - A folder is derived in place: each file with a filter gets `.md` beside it, nothing is renamed, and the files without a filter are named in the output. - `derived.md` starts with a fixed header (`derived_by`, `filter`, `engine`, `derived`, `derived_from`, `original_name` for a renamed file, `title`, then the fields of the format) followed by the derived text. The same bytes always give the same text; only `derived` differs between runs. `engine` names the tool version and, where a package reads the format, its version. -- Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, `.xlsm`), e-mail (`.eml`) and ZIP archives (`.zip`), read with the standard library only. +- Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, `.xlsm`), e-mail (`.eml`) and ZIP archives (`.zip`), read with the standard library only; PDF (`.pdf`, `pypdf`) and Outlook messages (`.msg`, `olefile`), each read by a package in `tools/requirements.txt` that is imported only when such a file is derived. A missing package refuses that format alone, with the preflight hint; `wikitool doctor` shows one `import-filter-` line per package-backed format. A PDF gets `` markers and a WARN when it yields almost no text (a scan; there is no OCR); an encrypted PDF is refused. A `.msg` is a container like an `.eml`; an embedded message inside it is named, not read. - A ZIP and a mail are containers. A ZIP at the top of `incoming/` becomes the folder `incoming//` holding its files under their own names, each with a `.md` where a filter reads it; the ZIP is removed and the output names it with its sha256. A mail becomes a folder with `original.eml`, `derived.md` and its attachments under their own names; a ZIP attached to it is unpacked once into a folder of its own, and an attached mail is quoted in `derived.md`, not unpacked. A ZIP inside a ZIP, an absolute or `..` path, a symbolic link, an encrypted entry, more than 1000 entries, a file over 25 MiB or more than 250 MiB in total refuse the whole call. Hidden entries (`__MACOSX/`, `.DS_Store`, anything starting with a dot) are named, not written. A name taken twice, or reserved (`original.*` and `derived.md` at the top of a mail's folder, `_capture.json` anywhere), gets `-2`, `-3` before its extension, and the output lists each rename. - A name already taken under `raw/` is a WARN, not a refusal: whether the folder is a later edition of that source or a separate one is the human's call. A later edition of a derived source is replaced file by file with `raw accept --replaces`; there is no `--replaces-bundle` for it. - After a success the output prints the warnings the filters raised, at most three `CHECK` lines per format that was derived in this run - what to look at before the text is trusted - and the `raw accept` line for the result. diff --git a/tools/CONTRACT.md b/tools/CONTRACT.md index 88f1bfa..52711b1 100644 --- a/tools/CONTRACT.md +++ b/tools/CONTRACT.md @@ -1515,7 +1515,7 @@ Derive reading text from a PDF, e-mail, Office file or web page in `incoming/`, - A single file directly in `incoming/` becomes a folder named after its stem (or `--name`) holding `original.` - the original bytes, moved - and `derived.md`. The fixed names keep the path inside the path budget; the header of `derived.md` records the file's own name as `original_name`. - A folder is derived in place: each file with a filter gets `.md` beside it, nothing is renamed, and the files without a filter are named in the output. - `derived.md` starts with a fixed header (`derived_by`, `filter`, `engine`, `derived`, `derived_from`, `original_name` for a renamed file, `title`, then the fields of the format) followed by the derived text. The same bytes always give the same text; only `derived` differs between runs. `engine` names the tool version and, where a package reads the format, its version. -- Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, `.xlsm`), e-mail (`.eml`) and ZIP archives (`.zip`), read with the standard library only. +- Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, `.xlsm`), e-mail (`.eml`) and ZIP archives (`.zip`), read with the standard library only; PDF (`.pdf`, `pypdf`) and Outlook messages (`.msg`, `olefile`), each read by a package in `tools/requirements.txt` that is imported only when such a file is derived. A missing package refuses that format alone, with the preflight hint; `wikitool doctor` shows one `import-filter-` line per package-backed format. A PDF gets `` markers and a WARN when it yields almost no text (a scan; there is no OCR); an encrypted PDF is refused. A `.msg` is a container like an `.eml`; an embedded message inside it is named, not read. - A ZIP and a mail are containers. A ZIP at the top of `incoming/` becomes the folder `incoming//` holding its files under their own names, each with a `.md` where a filter reads it; the ZIP is removed and the output names it with its sha256. A mail becomes a folder with `original.eml`, `derived.md` and its attachments under their own names; a ZIP attached to it is unpacked once into a folder of its own, and an attached mail is quoted in `derived.md`, not unpacked. A ZIP inside a ZIP, an absolute or `..` path, a symbolic link, an encrypted entry, more than 1000 entries, a file over 25 MiB or more than 250 MiB in total refuse the whole call. Hidden entries (`__MACOSX/`, `.DS_Store`, anything starting with a dot) are named, not written. A name taken twice, or reserved (`original.*` and `derived.md` at the top of a mail's folder, `_capture.json` anywhere), gets `-2`, `-3` before its extension, and the output lists each rename. - A name already taken under `raw/` is a WARN, not a refusal: whether the folder is a later edition of that source or a separate one is the human's call. A later edition of a derived source is replaced file by file with `raw accept --replaces`; there is no `--replaces-bundle` for it. - After a success the output prints the warnings the filters raised, at most three `CHECK` lines per format that was derived in this run - what to look at before the text is trusted - and the `raw accept` line for the result. @@ -3692,6 +3692,7 @@ Check that this instance is correctly configured. **NOTES** - Checks dependencies (Python, ripgrep), author resolution, stack version, git identity/branch/remote, published skills, the kb/raw/reports/work/instructions structure, and generated files. +- Import filters (`import-filter-`): one line for each `raw derive` format that reads through a package (PDF, Outlook `.msg`) - a `WARN` naming the package when it is missing, since that costs only its own format; fixed by running `tools/preflight.sh` again. - Tool paths (`tool-paths`): `.wikitool-tools.json` written by a preflight that finished, every tool `tools/prerequisites.txt` names for this platform recorded, and every recorded path still there - a `FAIL` otherwise, fixed by running `tools/preflight.sh` again (PowerShell 7: `tools/preflight.ps1`). - Install folder (`install-dir`): on Windows with long paths off, a `FAIL` when the folder holding `tools/` is longer than `tools/prerequisites.txt` allows (95 characters) - the limit the preflight enforces before it sets anything up. - PowerShell (`execution-policy`, `script-marks`; Windows only, `OK` elsewhere): a `FAIL` when the effective execution policy is `Restricted` or `AllSigned` - the line then says whether a group policy sets it, which only whoever administers the computer can change - and a `FAIL` when a script under `tools/` carries a Mark of the Web from the internet zone, which a browser download unpacked in Explorer leaves behind and `Invoke-WebRequest` plus `tar` do not. `tools/preflight.ps1` checks the same two things before it stops. diff --git a/tools/README.md b/tools/README.md index 37a1e84..d2cdd5b 100644 --- a/tools/README.md +++ b/tools/README.md @@ -66,7 +66,9 @@ excepted. `jsonschema` and `PyYAML` are hard dependencies, not optional extras: schema validation is the tool's whole safety net, so `cli.py` fails loudly with the -fix rather than degrading silently. +fix rather than degrading silently. `pypdf` and `olefile` are in `requirements.txt` too +but are imported only when a PDF or an Outlook `.msg` is derived, so a missing one stops +that format alone; `doctor` reports each. ## Layout @@ -112,7 +114,7 @@ tools/ kb_state.py the KB version (.wikitool-kb.json) and the migration chain corpus_diff.py invariant comparison of kb/ between two revisions web_capture.py `raw fetch`'s core: fetch a page, decide its charset, derive Markdown-like text from the HTML - standard library only, deterministic - filters/ the import filters behind `raw derive`: a registry (`__init__.py`) and one module per format family (HTML, mail, Office, ZIP), each registering the extensions it reads - standard library, or a package imported lazily so a missing one fails only its own format + filters/ the import filters behind `raw derive`: a registry (`__init__.py`) and one module per format family (HTML, mail, Outlook MSG, Office, PDF, ZIP), each registering the extensions it reads - standard library, or a package imported lazily so a missing one fails only its own format repo_capture.py `raw capture`/`raw status`'s core: resolve a ref rule, fetch it into a bare cache, select files by glob as blobs, the `_capture.json` manifest - git with the host's credentials, never a prompt; plus the one write back, a single-file commit pushed without force for `export guidelines` guideline_export.py `export guidelines`' core: select pages by `search` predicates, render them into one deterministic GUIDELINES.md, and recognise an export again (`EXPORT_MARKER`) - with no CLI attached search/ pluggable search backends, plus service.py - the search core diff --git a/tools/chemenu/commands/doctor.py b/tools/chemenu/commands/doctor.py index 94b9746..f4f6954 100644 --- a/tools/chemenu/commands/doctor.py +++ b/tools/chemenu/commands/doctor.py @@ -21,7 +21,7 @@ from typing import Optional import typer from rich.console import Console -from chemenu import cli_contract, config, config_shapes, conventions, kb_collections, prerequisites, toolpaths, version as version_mod +from chemenu import cli_contract, config, config_shapes, conventions, filters, kb_collections, prerequisites, toolpaths, version as version_mod from chemenu.commands import git_publish, instructions_cmd from chemenu.commands._util import rel_path from chemenu.session import ENV_VAR as SESSION_ENV_VAR @@ -76,6 +76,26 @@ def check_ripgrep() -> Check: ) +def check_import_filters() -> list[Check]: + """One line per `raw derive` filter that needs a package: a missing one costs only its + own format, so it is a WARN, never a FAIL.""" + found = [] + for candidate in filters.all_filters(): + if not candidate.requires: + continue + missing = candidate.missing() + formats = ", ".join(candidate.extensions) + if missing: + found.append(Check( + f"import-filter-{candidate.name}", "WARN", + f"{', '.join(missing)} missing - raw derive cannot read {formats} until it is installed", + PREFLIGHT_FIX, + )) + else: + found.append(Check(f"import-filter-{candidate.name}", "OK", f"{candidate.engine().split(' + ', 1)[1]} reads {formats}")) + return found + + def check_tool_paths() -> Check: """`.wikitool-tools.json`: written by a preflight that finished, and every path in it still there. The launcher refuses to start without a complete @@ -742,6 +762,7 @@ def run_doctor() -> list[Check]: check_python(), check_tool_paths(), check_ripgrep(), + *check_import_filters(), check_install_dir(), check_execution_policy(), check_script_marks(), @@ -779,6 +800,9 @@ def run_doctor() -> list[Check]: "Checks dependencies (Python, ripgrep), author resolution, stack version, git " "identity/branch/remote, published skills, the kb/raw/reports/work/instructions " "structure, and generated files.", + "Import filters (`import-filter-`): one line for each `raw derive` format that " + "reads through a package (PDF, Outlook `.msg`) - a `WARN` naming the package when it is " + "missing, since that costs only its own format; fixed by running `tools/preflight.sh` again.", "Tool paths (`tool-paths`): `.wikitool-tools.json` written by a preflight that " "finished, every tool `tools/prerequisites.txt` names for this platform recorded, and " "every recorded path still there - a `FAIL` otherwise, fixed by running " diff --git a/tools/chemenu/commands/raw_cmd.py b/tools/chemenu/commands/raw_cmd.py index 8564930..9a13d97 100644 --- a/tools/chemenu/commands/raw_cmd.py +++ b/tools/chemenu/commands/raw_cmd.py @@ -2135,7 +2135,13 @@ def _derive_next(target: Path) -> str: "reads the format, its version.", "Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, " "`.xlsm`), e-mail (`.eml`) and ZIP archives (`.zip`), read with the standard library " - "only.", + "only; PDF (`.pdf`, `pypdf`) and Outlook messages (`.msg`, `olefile`), each read by a " + "package in `tools/requirements.txt` that is imported only when such a file is derived. " + "A missing package refuses that format alone, with the preflight hint; `wikitool " + "doctor` shows one `import-filter-` line per package-backed format. A PDF gets " + "`` markers and a WARN when it yields almost no text (a scan; there is " + "no OCR); an encrypted PDF is refused. A `.msg` is a container like an `.eml`; an " + "embedded message inside it is named, not read.", "A ZIP and a mail are containers. A ZIP at the top of `incoming/` becomes the folder " "`incoming//` holding its files under their own names, each with a `.md` " "where a filter reads it; the ZIP is removed and the output names it with its sha256. A " diff --git a/tools/chemenu/filters/__init__.py b/tools/chemenu/filters/__init__.py index 9a50d2d..078fe5e 100644 --- a/tools/chemenu/filters/__init__.py +++ b/tools/chemenu/filters/__init__.py @@ -205,4 +205,4 @@ def unique(lines: Iterable[str]) -> list[str]: return out -from chemenu.filters import archive, html_page, mail, ooxml # noqa: E402,F401 (register themselves) +from chemenu.filters import archive, html_page, mail, msg, ooxml, pdf # noqa: E402,F401 (register themselves) diff --git a/tools/chemenu/filters/mail.py b/tools/chemenu/filters/mail.py index 046fd8f..5e54a62 100644 --- a/tools/chemenu/filters/mail.py +++ b/tools/chemenu/filters/mail.py @@ -1,4 +1,5 @@ -"""E-mail: `.eml` (RFC 5322 / MIME), read with the standard library.""" +"""E-mail: `.eml` (RFC 5322 / MIME), read with the standard library. The Outlook +`.msg` filter (`msg.py`) shares everything after the parsing.""" from __future__ import annotations import re @@ -26,7 +27,7 @@ class Attachment: @property def is_message(self) -> bool: - return self.content_type == "message/rfc822" or self.name.lower().endswith(".eml") + return self.content_type == "message/rfc822" or self.name.lower().endswith((".eml", ".msg")) @dataclass @@ -178,6 +179,14 @@ def _quote_section(inner: ParsedMail) -> str: return "\n".join(lines) +def read_attached(attachment: Attachment) -> ParsedMail: + if attachment.name.lower().endswith(".msg"): + from chemenu.filters import msg # imports this module; kept lazy + + return msg.MsgFilter().read(attachment.data) + return parse(attachment.data) + + def _subject(parsed: ParsedMail) -> str: subject = parsed.headers["subject"] return subject if isinstance(subject, str) else "" @@ -190,8 +199,11 @@ class EmlFilter(Filter): name = "eml" extensions = (".eml",) + def read(self, data: bytes) -> ParsedMail: + return parse(data) + def derive(self, data: bytes, name: str) -> Derivation: - parsed = parse(data) + parsed = self.read(data) warnings = list(parsed.warnings) if parsed.attachments: warnings.append( @@ -207,7 +219,7 @@ class EmlFilter(Filter): ) def container(self, data: bytes, name: str) -> Optional[Container]: - parsed = parse(data) + parsed = self.read(data) collector = Collector(reserve_top=True) sections: list[str] = [] warnings = list(parsed.warnings) @@ -215,9 +227,10 @@ class EmlFilter(Filter): file_name = clean_name(attachment.name, f"attachment-{number}") if attachment.is_message: try: - inner = parse(attachment.data) - except ValidationError: + inner = read_attached(attachment) + except ValidationError as exc: collector.add_file(file_name, attachment.data) + warnings.append(f"The attachment {file_name!r} looks like a message but is written as a file: {exc}") continue sections.append(_quote_section(inner)) if inner.attachments: diff --git a/tools/chemenu/filters/msg.py b/tools/chemenu/filters/msg.py new file mode 100644 index 0000000..4001d73 --- /dev/null +++ b/tools/chemenu/filters/msg.py @@ -0,0 +1,213 @@ +"""Outlook `.msg` (a compound file), read with `olefile` (imported when one is derived). + +Only what a reader of the mail needs: the headers, the body (plain text, else HTML) +and the attachments by name and bytes. The result is the same `ParsedMail` an `.eml` +yields, so everything after the parsing is shared with `mail.py`. +""" +from __future__ import annotations + +import datetime +import io +import re +import struct +from email import message_from_bytes, policy +from email.utils import parsedate_to_datetime +from typing import Optional + +from chemenu import web_capture +from chemenu.errors import ValidationError +from chemenu.filters import register +from chemenu.filters.mail import Attachment, EmlFilter, ParsedMail + +_SUBJECT = "0037" +_BODY = "1000" +_HTML = "1013" +_RTF = "1009" +_HEADERS = "007D" +_SENDER_NAME = "0C1A" +_SENDER_ADDRESS = "5D01" +_SENDER_FALLBACK = "0C1F" +_DISPLAY_TO = "0E04" +_DISPLAY_CC = "0E03" +_ATT_LONG_NAME = "3707" +_ATT_SHORT_NAME = "3704" +_ATT_DATA = "3701" +_ATT_MIME = "370E" +_ATT_CONTENT_ID = "3712" +_ATT_METHOD = "3705" +_METHOD_EMBEDDED_MESSAGE = 5 +_FILETIME_EPOCH = datetime.datetime(1601, 1, 1, tzinfo=datetime.timezone.utc) + + +def _norm_header(text: str) -> str: + return re.sub(r"\s+", " ", text).strip() + + +class _Message: + """The streams of one message (or of one attached message storage) by property.""" + + def __init__(self, ole, prefix: str = "") -> None: + self._ole = ole + self._prefix = prefix + + def _path(self, tag: str, kind: str) -> str: + return f"{self._prefix}__substg1.0_{tag}{kind}" + + def raw(self, tag: str) -> Optional[bytes]: + for kind in ("001F", "001E", "0102"): + path = self._path(tag, kind) + if self._ole.exists(path): + return self._ole.openstream(path).read() + return None + + def text(self, tag: str) -> str: + for kind, encoding in (("001F", "utf-16-le"), ("001E", "cp1252")): + path = self._path(tag, kind) + if self._ole.exists(path): + return self._ole.openstream(path).read().decode(encoding, errors="replace").strip("\x00").strip() + return "" + + def _properties(self) -> dict[tuple[int, int], bytes]: + """The fixed-size properties from `__properties_version1.0`, by (id, type). Its + header is 32 bytes for the message itself and 8 for an attachment.""" + path = f"{self._prefix}__properties_version1.0" + if not self._ole.exists(path): + return {} + data = self._ole.openstream(path).read() + found: dict[tuple[int, int], bytes] = {} + for offset in range(32 if not self._prefix else 8, len(data) - 15, 16): + (tag,) = struct.unpack_from("> 16, tag & 0xFFFF)] = data[offset + 8 : offset + 16] + return found + + def integer(self, tag: str) -> Optional[int]: + value = self._properties().get((int(tag, 16), 0x0003)) + return None if value is None else struct.unpack_from(" str: + value = self._properties().get((int(tag, 16), 0x0040)) + if value is None: + return "" + try: + return (_FILETIME_EPOCH + datetime.timedelta(microseconds=struct.unpack(" str: + if not headers: + return "" + parsed = message_from_bytes(headers.encode("utf-8", errors="replace"), policy=policy.default) + try: + return _norm_header(str(parsed[name] or "")) + except Exception: + return "" + + +def _header_date(headers: str) -> str: + value = _header_value(headers, "date") + try: + return parsedate_to_datetime(value).isoformat() if value else "" + except (TypeError, ValueError): + return value + + +def _address_list(text: str) -> list[str]: + return [part.strip() for part in text.split(";") if part.strip()] + + +def _html_text(raw: bytes) -> str: + for encoding in ("utf-8", "cp1252"): + try: + html = raw.decode(encoding) + break + except UnicodeDecodeError: + continue + else: + html = raw.decode("utf-8", errors="replace") + return web_capture.derive_text(html, "").strip() + + +def _body(message: _Message, out: ParsedMail) -> None: + plain = message.text(_BODY) + html = message.raw(_HTML) + if plain: + out.body = plain.replace("\r\n", "\n").replace("\r", "\n").strip() + elif html: + out.body = _html_text(html) + else: + what = "has only an RTF body" if message.raw(_RTF) else "has no body" + out.warnings.append(f"The message {what}, which raw derive does not read; the text is empty.") + + +def _read(ole) -> ParsedMail: + message = _Message(ole) + out = ParsedMail() + headers = message.text(_HEADERS) + sender = message.text(_SENDER_NAME) + address = message.text(_SENDER_ADDRESS) or message.text(_SENDER_FALLBACK) + if sender and address and sender != address: + sender = f"{sender} <{address}>" + date = _header_date(headers) or message.time("0039") or message.time("0E06") + out.headers = { + "from": _header_value(headers, "from") or sender or address, + "to": _address_list(message.text(_DISPLAY_TO)), + "cc": _address_list(message.text(_DISPLAY_CC)), + "date": date, + "subject": message.text(_SUBJECT), + "message_id": _header_value(headers, "message-id"), + } + _body(message, out) + + inline = 0 + for entry in sorted( + {tuple(path[:1]) for path in ole.listdir(streams=False, storages=True) if path[0].startswith("__attach_version1.0_")} + ): + storage = f"{entry[0]}/" + attachment = _Message(ole, storage) + name = attachment.text(_ATT_LONG_NAME) or attachment.text(_ATT_SHORT_NAME) or f"attachment-{len(out.attachments) + inline + 1}" + if attachment.integer(_ATT_METHOD) == _METHOD_EMBEDDED_MESSAGE: + out.warnings.append( + f"An embedded message {name!r} is named but not read; open the original for its content." + ) + continue + content = attachment.raw(_ATT_DATA) or b"" + if attachment.text(_ATT_CONTENT_ID) and attachment.text(_ATT_MIME).startswith("image/"): + inline += 1 + continue + out.attachments.append(Attachment(name, content, attachment.text(_ATT_MIME) or "application/octet-stream")) + out.inline_images = inline + if inline: + out.warnings.append(f"{inline} inline image(s) are not in the text.") + return out + + +class MsgFilter(EmlFilter): + """An Outlook message: the same text, header and container rules as an `.eml`.""" + + name = "msg" + extensions = (".msg",) + requires = ("olefile",) + + def read(self, data: bytes) -> ParsedMail: + import olefile + + if not data.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"): + raise ValidationError("Not an Outlook message: the file is not a compound document.") + try: + ole = olefile.OleFileIO(io.BytesIO(data)) + except (OSError, olefile.OleFileError) as exc: + raise ValidationError(f"Not a readable Outlook message: {exc}.") from None + try: + if not ole.exists("__properties_version1.0"): + raise ValidationError("Not an Outlook message: the compound document has no message properties.") + return _read(ole) + except ValidationError: + raise + except (OSError, struct.error, ValueError, IndexError, KeyError) as exc: + raise ValidationError(f"Not a readable Outlook message: {exc}.") from None + finally: + ole.close() + + +register(MsgFilter()) diff --git a/tools/chemenu/filters/pdf.py b/tools/chemenu/filters/pdf.py new file mode 100644 index 0000000..c716d0a --- /dev/null +++ b/tools/chemenu/filters/pdf.py @@ -0,0 +1,60 @@ +"""PDF, read with `pypdf` (imported when a PDF is derived, never at start-up).""" +from __future__ import annotations + +import io + +from chemenu.errors import ValidationError +from chemenu.filters import Derivation, Filter, register + +SCAN_CHARS_PER_PAGE = 50 + + +class PdfFilter(Filter): + name = "pdf" + extensions = (".pdf",) + requires = ("pypdf",) + + def derive(self, data: bytes, name: str) -> Derivation: + import pypdf + from pypdf import errors + + try: + reader = pypdf.PdfReader(io.BytesIO(data)) + if reader.is_encrypted: + raise ValidationError( + f"{name} is encrypted; raw derive does not read it. Save an unprotected copy " + "into incoming/ and derive that." + ) + pages = [(page.extract_text() or "").strip() for page in reader.pages] + title = "" + if reader.metadata is not None and reader.metadata.title: + title = " ".join(str(reader.metadata.title).split()) + except ValidationError: + raise + except (errors.PyPdfError, ValueError, KeyError, TypeError, RecursionError, OSError) as exc: + raise ValidationError(f"{name} is not a readable PDF: {exc}.") from None + if not pages: + raise ValidationError(f"{name} has no pages.") + + blocks = [f"\n{text}".rstrip() for number, text in enumerate(pages, 1)] + characters = sum(len(text) for text in pages) + warnings: list[str] = [] + checks = [ + "Page breaks are the markers; tables and multi-column layouts may " + "come out in reading order, not as laid out - check figures against the original." + ] + if characters < SCAN_CHARS_PER_PAGE * len(pages): + warnings.append( + f"The PDF yields {characters} character(s) over {len(pages)} page(s): it is probably " + "a scan. raw derive does no OCR, so the text is missing or incomplete." + ) + return Derivation( + text="\n\n".join(blocks), + title=title, + fields=(("pages", str(len(pages))),), + warnings=tuple(warnings), + checks=tuple(checks), + ) + + +register(PdfFilter()) diff --git a/tools/chemenu/tests/binary_formats.py b/tools/chemenu/tests/binary_formats.py new file mode 100644 index 0000000..d205884 --- /dev/null +++ b/tools/chemenu/tests/binary_formats.py @@ -0,0 +1,224 @@ +"""Builders for the two binary formats `raw derive` reads with a package: a PDF and an +Outlook `.msg` (a compound file). Built here, byte by byte, so no binary sits in the +repository and every fixture is readable in the test that uses it. Whether each one is a +valid document is settled by the package that reads it in the tests.""" +from __future__ import annotations + +import struct +from typing import Optional + +END, FREE, FATSECT = 0xFFFFFFFE, 0xFFFFFFFF, 0xFFFFFFFD + + +def build_pdf(pages: list[list[str]], *, title: Optional[str] = None) -> bytes: + """A PDF with one text line per list item; an empty page is an empty content stream.""" + objects: dict[int, bytes] = {} + page_ids = [3 + 2 * n for n in range(len(pages))] + font_id = 3 + 2 * len(pages) + info_id = font_id + 1 + objects[1] = b"<< /Type /Catalog /Pages 2 0 R >>" + kids = " ".join(f"{page} 0 R" for page in page_ids) + objects[2] = f"<< /Type /Pages /Kids [{kids}] /Count {len(pages)} >>".encode() + for page_id, lines in zip(page_ids, pages): + ops = ["BT /F1 12 Tf 72 720 Td 14 TL"] + for line in lines: + escaped = line.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + ops.append(f"({escaped}) Tj T*") + ops.append("ET") + stream = "\n".join(ops).encode("latin-1") if lines else b"" + objects[page_id] = ( + f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents {page_id + 1} 0 R " + f"/Resources << /Font << /F1 {font_id} 0 R >> >> >>" + ).encode() + objects[page_id + 1] = b"<< /Length %d >>\nstream\n" % len(stream) + stream + b"\nendstream" + objects[font_id] = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>" + trailer = "/Root 1 0 R" + if title is not None: + objects[info_id] = f"<< /Title ({title}) >>".encode("latin-1") + trailer += f" /Info {info_id} 0 R" + out = bytearray(b"%PDF-1.4\n") + offsets = {} + for number in sorted(objects): + offsets[number] = len(out) + out += f"{number} 0 obj\n".encode() + objects[number] + b"\nendobj\n" + xref = len(out) + size = max(objects) + 1 + out += f"xref\n0 {size}\n".encode() + b"0000000000 65535 f \n" + for number in range(1, size): + out += f"{offsets[number]:010d} 00000 n \n".encode() + out += f"trailer\n<< /Size {size} {trailer} >>\nstartxref\n{xref}\n%%EOF\n".encode() + return bytes(out) + + +class _Node: + def __init__(self, name: str, kind: int, data: bytes = b"") -> None: + self.name, self.kind, self.data = name, kind, data + self.children: dict[str, "_Node"] = {} + self.index = 0 + self.start = END + self.size = 0 + + +def build_cfb(files: dict[str, bytes]) -> bytes: + """A compound file (version 3) holding `files` (path with `/` -> bytes). Streams under + 4096 bytes go into the mini stream, as the format requires.""" + root = _Node("Root Entry", 5) + for path, data in files.items(): + *folders, leaf = path.split("/") + node = root + for folder in folders: + node = node.children.setdefault(folder, _Node(folder, 1)) + node.children[leaf] = _Node(leaf, 2, data) + order: list[_Node] = [] + + def number(node: _Node) -> None: + node.index = len(order) + order.append(node) + for child in node.children.values(): + number(child) + + number(root) + streams = [n for n in order if n.kind == 2] + small = [n for n in streams if 0 < len(n.data) < 4096] + big = [n for n in streams if len(n.data) >= 4096] + + mini = bytearray() + mini_chain: list[int] = [] + for node in small: + node.start = len(mini) // 64 + node.size = len(node.data) + blocks = -(-len(node.data) // 64) + mini += node.data.ljust(blocks * 64, b"\x00") + mini_chain += [node.start + i + 1 for i in range(blocks - 1)] + [END] + for node in streams: + node.size = len(node.data) + + n_minifat = -(-len(mini_chain) // 128) + n_dir = -(-len(order) // 4) + n_mini = -(-len(mini) // 512) + big_sectors = {id(n): -(-len(n.data) // 512) for n in big} + total = 1 + n_minifat + n_dir + n_mini + sum(big_sectors.values()) + assert total <= 128, "the builder writes one FAT sector" + + fat = [FREE] * 128 + fat[0] = FATSECT + cursor = 1 + first_minifat = cursor if n_minifat else END + for i in range(n_minifat): + fat[cursor + i] = cursor + i + 1 if i < n_minifat - 1 else END + cursor += n_minifat + first_dir = cursor + for i in range(n_dir): + fat[cursor + i] = cursor + i + 1 if i < n_dir - 1 else END + cursor += n_dir + root.start = cursor if n_mini else END + root.size = len(mini) + for i in range(n_mini): + fat[cursor + i] = cursor + i + 1 if i < n_mini - 1 else END + cursor += n_mini + for node in big: + node.start = cursor + count = big_sectors[id(node)] + for i in range(count): + fat[cursor + i] = cursor + i + 1 if i < count - 1 else END + cursor += count + + def entry(node: Optional[_Node], right: int = FREE) -> bytes: + if node is None: + return struct.pack("<64sHBBIII16sIQQIQ", b"", 0, 0, 0, FREE, FREE, FREE, b"", 0, 0, 0, 0, 0) + name = node.name.encode("utf-16-le") + kids = list(node.children.values()) + return struct.pack( + "<64sHBBIII16sIQQIQ", name.ljust(64, b"\x00"), len(name) + 2, node.kind, 1, FREE, right, + kids[0].index if kids else FREE, b"", 0, 0, 0, node.start, node.size, + ) + + rights: dict[int, int] = {} + for node in order: + kids = list(node.children.values()) + for current, following in zip(kids, kids[1:]): + rights[current.index] = following.index + directory = b"".join(entry(n, rights.get(n.index, FREE)) for n in order) + empty_entry = entry(None) + directory = directory[: len(order) * 128] + empty_entry * (n_dir * 4 - len(order)) + + header = struct.pack( + "<8s16sHHHHH6sIIIIIIIII", b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", b"", 0x3E, 3, 0xFFFE, 9, 6, b"", 0, 1, + first_dir, 0, 4096, first_minifat, n_minifat, END, 0, + ) + struct.pack("<109I", 0, *([FREE] * 108)) + + minifat = struct.pack(f"<{n_minifat * 128}I", *(mini_chain + [FREE] * (n_minifat * 128 - len(mini_chain)))) + out = bytearray(header) + out += struct.pack("<128I", *fat) + out += minifat + out += directory + out += bytes(mini).ljust(n_mini * 512, b"\x00") + for node in big: + out += node.data.ljust(big_sectors[id(node)] * 512, b"\x00") + return bytes(out) + + +def _utf16(text: str) -> bytes: + return text.encode("utf-16-le") + + +def build_msg( + *, + subject: str = "", + body: Optional[str] = None, + html: Optional[str] = None, + rtf: Optional[bytes] = None, + sender: str = "", + sender_address: str = "", + to: str = "", + cc: str = "", + attachments: Optional[list[tuple[str, bytes]]] = None, + inline_images: Optional[list[tuple[str, bytes]]] = None, + embedded: Optional[list[str]] = None, + sent: Optional[int] = None, +) -> bytes: + """An Outlook message. `sent` is a FILETIME (100 ns ticks since 1601).""" + files: dict[str, bytes] = {} + + def text(prefix: str, tag: str, value: str) -> None: + if value: + files[f"{prefix}__substg1.0_{tag}001F"] = _utf16(value) + + text("", "0037", subject) + text("", "0C1A", sender) + text("", "5D01", sender_address) + text("", "0E04", to) + text("", "0E03", cc) + if body is not None: + text("", "1000", body) + if html is not None: + files["__substg1.0_10130102"] = html.encode("utf-8") + if rtf is not None: + files["__substg1.0_10090102"] = rtf + props = b"\x00" * 32 + if sent is not None: + props += struct.pack("") < text.index("Second line (with brackets)") < text.index("") + assert "WARN" not in out + assert out.count("CHECK") == 1 and "page" in out + + +@needs_pypdf +def test_a_pdf_without_text_is_derived_with_a_scan_warning(tree, capsys): + _put(tree, "scan.pdf", build_pdf([[], [], []])) + _derive("incoming/scan.pdf") + out = capsys.readouterr().out + assert "WARN" in out and "scan" in out and "3 page" in out + assert (tree / "incoming/scan/original.pdf").is_file() + + +@needs_pypdf +def test_a_pdf_that_is_not_a_pdf_is_refused_untouched(tree): + _put(tree, "fake.pdf", b"this is not a pdf at all") + before = _snapshot(tree / "incoming") + _refused("incoming/fake.pdf") + assert _snapshot(tree / "incoming") == before + + +@needs_pypdf +def test_an_encrypted_pdf_is_refused_untouched(tree, capsys): + import pypdf + + writer = pypdf.PdfWriter(clone_from=pypdf.PdfReader(io.BytesIO(build_pdf([[LONG_LINE]])))) + try: + writer.encrypt("secret", algorithm="RC4-128") + except Exception as exc: # no crypto provider for the algorithm on this machine + pytest.skip(f"cannot build an encrypted PDF here: {exc}") + buffer = io.BytesIO() + writer.write(buffer) + _put(tree, "locked.pdf", buffer.getvalue()) + before = _snapshot(tree / "incoming") + _refused("incoming/locked.pdf") + assert "encrypted" in capsys.readouterr().out + assert _snapshot(tree / "incoming") == before + + +@needs_pypdf +def test_a_pdf_inside_a_zip_gets_its_md(tree): + _put(tree, "bundle.zip", make_zip({"a.pdf": build_pdf([[LONG_LINE]]), "b.txt": b"x"})) + _derive("incoming/bundle.zip") + folder = tree / "incoming/bundle" + assert LONG_LINE in (folder / "a.md").read_text(encoding="utf-8") + assert (folder / "a.pdf").is_file() and not (folder / "b.md").exists() + + +@needs_pypdf +def test_a_pdf_attached_to_a_mail_gets_its_md(tree): + _put(tree, "mail.eml", make_mail([("offer.pdf", build_pdf([[LONG_LINE]]), "application/pdf")])) + _derive("incoming/mail.eml") + assert LONG_LINE in (tree / "incoming/mail/offer.md").read_text(encoding="utf-8") + + +def test_a_missing_package_stops_only_its_own_format(tree, monkeypatch, capsys): + pdf_filter = filters.filter_for("a.pdf") + monkeypatch.setattr(type(pdf_filter), "missing", lambda self: ["pypdf"]) + _put(tree, "a.pdf", b"%PDF-1.4") + _put(tree, "b.docx", make_docx("Still works")) + before = _snapshot(tree / "incoming") + _refused("incoming/a.pdf") + out = capsys.readouterr().out + assert "pypdf" in out and "preflight" in out + assert _snapshot(tree / "incoming") == before + _derive("incoming/b.docx") + assert "Still works" in (tree / "incoming/b/derived.md").read_text(encoding="utf-8") + + +def test_a_package_missing_inside_a_zip_refuses_the_whole_zip(tree, monkeypatch): + monkeypatch.setattr(type(filters.filter_for("a.pdf")), "missing", lambda self: ["pypdf"]) + _put(tree, "bundle.zip", make_zip({"a.pdf": b"%PDF-1.4", "b.txt": b"x"})) + before = _snapshot(tree / "incoming") + _refused("incoming/bundle.zip") + assert _snapshot(tree / "incoming") == before + + +# --- Outlook .msg -------------------------------------------------------------- + + +SENT = 134_000_000_000_000_000 # 2025-08-18T14:13:20+00:00 + + +@needs_olefile +def test_a_msg_becomes_a_folder_like_a_mail(tree, capsys): + data = [b"attachment bytes" * 400, b"small"] + original = build_msg( + subject="Unterlagen", body="Hello Bob\r\nSee attached.", sender="Anna", sender_address="anna@example.org", + to="bob@example.org; carol@example.org", sent=SENT, + attachments=[("report.bin", data[0]), ("note.txt", data[1])], + ) + _put(tree, "Mail.msg", original) + _derive("incoming/Mail.msg") + out = capsys.readouterr().out + + folder = tree / "incoming/Mail" + assert (folder / "original.msg").read_bytes() == original + assert (folder / "report.bin").read_bytes() == data[0] + assert (folder / "note.txt").read_bytes() == data[1] + header, text = _header_and_text(folder / "derived.md") + assert header["filter"] == "msg" and "olefile" in header["engine"] + assert header["from"] == "Anna " + assert header["to"] == ["bob@example.org", "carol@example.org"] + assert header["subject"] == "Unterlagen" and str(header["date"]).startswith("2025-08-18") + assert header["attachments"] == ["report.bin", "note.txt"] + assert text.strip() == "Hello Bob\nSee attached." + assert "report.bin" in out + + +@needs_olefile +def test_a_msg_with_only_html_takes_the_html_text(tree): + _put(tree, "h.msg", build_msg(subject="S", html="

Only html here

")) + _derive("incoming/h.msg") + _header, text = _header_and_text(tree / "incoming/h/derived.md") + assert "Only" in text and "html here" in text and "

" not in text + + +@needs_olefile +def test_a_msg_with_only_rtf_warns_and_has_no_text(tree, capsys): + _put(tree, "r.msg", build_msg(subject="S", rtf=b"{\\rtf1 hello}")) + _derive("incoming/r.msg") + out = capsys.readouterr().out + assert "WARN" in out and "RTF" in out + _header, text = _header_and_text(tree / "incoming/r/derived.md") + assert text.strip() == "" + + +@needs_olefile +def test_a_msg_names_inline_images_and_embedded_messages_without_writing_them(tree, capsys): + _put(tree, "m.msg", build_msg( + subject="S", body="text", inline_images=[("logo.png", b"png")], embedded=["forwarded.msg"], + attachments=[("real.bin", b"x")], + )) + _derive("incoming/m.msg") + out = capsys.readouterr().out + assert sorted(p.name for p in (tree / "incoming/m").iterdir()) == ["derived.md", "original.msg", "real.bin"] + assert "inline image" in out and "forwarded.msg" in out + + +@needs_olefile +def test_a_msg_attached_to_a_mail_is_quoted(tree, capsys): + inner = build_msg(subject="Inner subject", body="Inner words", sender="Dora", to="x@example.org") + _put(tree, "mail.eml", make_mail([("inner.msg", inner, "application/vnd.ms-outlook")])) + _derive("incoming/mail.eml") + folder = tree / "incoming/mail" + assert sorted(p.name for p in folder.iterdir()) == ["derived.md", "original.eml"] + _header, text = _header_and_text(folder / "derived.md") + assert "## Attached message: Inner subject" in text and "> Inner words" in text + + +@needs_olefile +def test_a_file_that_is_not_a_msg_is_refused_untouched(tree): + _put(tree, "x.msg", b"plain text pretending to be a message") + before = _snapshot(tree / "incoming") + _refused("incoming/x.msg") + assert _snapshot(tree / "incoming") == before + + +@needs_olefile +def test_a_compound_file_that_is_not_a_message_is_refused(tree): + from chemenu.tests.binary_formats import build_cfb + + _put(tree, "x.msg", build_cfb({"WordDocument": b"abc"})) + _refused("incoming/x.msg") + assert not (tree / "incoming/x").exists() + + +def test_a_msg_without_the_package_is_refused_and_a_docx_still_works(tree, monkeypatch, capsys): + monkeypatch.setattr(type(filters.filter_for("a.msg")), "missing", lambda self: ["olefile"]) + _put(tree, "a.msg", b"\xd0\xcf\x11\xe0") + before = _snapshot(tree / "incoming") + _refused("incoming/a.msg") + assert "olefile" in capsys.readouterr().out + assert _snapshot(tree / "incoming") == before + + +# --- doctor --------------------------------------------------------------------- + + +def test_doctor_names_each_filter_that_needs_a_package(): + names = {check.name for check in doctor.check_import_filters()} + assert names == {"import-filter-pdf", "import-filter-msg"} + + +def test_doctor_warns_for_a_missing_package_and_never_fails(monkeypatch): + monkeypatch.setattr(type(filters.filter_for("a.pdf")), "missing", lambda self: ["pypdf"]) + checks = {check.name: check for check in doctor.check_import_filters()} + assert checks["import-filter-pdf"].status == "WARN" + assert "pypdf" in checks["import-filter-pdf"].detail and checks["import-filter-pdf"].fix + assert all(check.status != "FAIL" for check in checks.values()) + + +@needs_pypdf +def test_doctor_reports_an_installed_package_with_its_version(): + check = next(c for c in doctor.check_import_filters() if c.name == "import-filter-pdf") + assert check.status == "OK" and check.detail.startswith("pypdf ") and ".pdf" in check.detail diff --git a/tools/requirements.txt b/tools/requirements.txt index b3e78ac..37c08ab 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -4,3 +4,5 @@ rich>=13.0 jsonschema>=4.18 referencing>=0.30 icalendar>=6.0 +pypdf>=5.0 +olefile>=0.47