From 9c0bb6bfbf7cca1ca1fd5919dbb689edeefc7be2 Mon Sep 17 00:00:00 2001 From: Torben Nehmer Date: Thu, 8 Oct 2026 12:12:31 +0200 Subject: [PATCH] feat(raw): raw derive turns Office files and mail into a folder with the original and a derived text Adds the import-filter registry (tools/chemenu/filters), the raw derive command with folder form, path-budget and name checks, DOCX/PPTX/XLSX and EML filters, the pending and accept hints, the wiki-ingest skill step and the raw/CONTRACT.md section. First of three cuts; containers and PDF/MSG follow. Files changed: - CHANGES.md - VERSION - docs/how-to/ingest-a-source.md - docs/reference/commands/README.md - docs/reference/commands/raw-material-and-uploads.md - instructions/wiki-ingest/SKILL.md - raw/CONTRACT.md - tools/CONTRACT.md - tools/README.md - tools/chemenu/cli_contract.py - tools/chemenu/commands/raw_cmd.py - tools/chemenu/filters/__init__.py - tools/chemenu/filters/html_page.py - tools/chemenu/filters/mail.py - tools/chemenu/filters/ooxml.py - tools/chemenu/tests/test_raw_derive.py - tools/chemenu/web_capture.py Co-Authored-By: Claude Code Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2 --- CHANGES.md | 3 +- VERSION | 2 +- docs/how-to/ingest-a-source.md | 4 + docs/reference/commands/README.md | 1 + .../commands/raw-material-and-uploads.md | 64 +- instructions/wiki-ingest/SKILL.md | 9 +- raw/CONTRACT.md | 45 +- tools/CONTRACT.md | 62 +- tools/README.md | 1 + tools/chemenu/cli_contract.py | 2 +- tools/chemenu/commands/raw_cmd.py | 385 +++++++++- tools/chemenu/filters/__init__.py | 172 +++++ tools/chemenu/filters/html_page.py | 34 + tools/chemenu/filters/mail.py | 177 +++++ tools/chemenu/filters/ooxml.py | 673 +++++++++++++++++ tools/chemenu/tests/test_raw_derive.py | 676 ++++++++++++++++++ tools/chemenu/web_capture.py | 11 +- 17 files changed, 2304 insertions(+), 17 deletions(-) create mode 100644 tools/chemenu/filters/__init__.py create mode 100644 tools/chemenu/filters/html_page.py create mode 100644 tools/chemenu/filters/mail.py create mode 100644 tools/chemenu/filters/ooxml.py create mode 100644 tools/chemenu/tests/test_raw_derive.py diff --git a/CHANGES.md b/CHANGES.md index 09d5ad8..0d59ca2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,7 +14,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. --- -## 9.0.0-beta.13 - 2026-10-07 - Gitea-Wiki als generierter Spiegel von docs/: Konverter, Push-Skript, Workflows +## 9.0.0-beta.14 - 2026-10-08 - raw derive reads PDF-free documents: Office files and mail become a folder with the original and a derived text **Author:** Torben Nehmer @@ -35,6 +35,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. - Die Formen der vier handgeschriebenen .wikitool-*.json stehen je in einem Datensatz (config_shapes), aus dem Lader, doctor und die Konfigurationsreferenz lesen - Gate-Liste: AGENTS.md § Gates ist die eine Liste (mit Rebase-Review Gate), cli_contract.GATES hält sie, docs verify vergleicht beide - Asset-Preflight: Verschieben aus dem Entpackordner wird bis ca. 30 s wiederholt, nennt die Systemmeldung und rollt nach endgültigem Scheitern zurück +- raw derive reads PDF-free documents: Office files and mail become a folder with the original and a derived text **Low impact** - version bump: Abschlussmeldung nennt Stichpunkt oder Themenabsatz statt pauschal Prosa diff --git a/VERSION b/VERSION index f1d18b5..72cb8d7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -9.0.0-beta.13 +9.0.0-beta.14 diff --git a/docs/how-to/ingest-a-source.md b/docs/how-to/ingest-a-source.md index 36086ee..7197dfa 100644 --- a/docs/how-to/ingest-a-source.md +++ b/docs/how-to/ingest-a-source.md @@ -16,6 +16,10 @@ pick a directory: - **A web page** needs no download of your own. Name the URL to the agent, and `tools/wikitool raw fetch` puts two files into `incoming/`: the HTML exactly as received, and a text derived from it with a header recording where and when it was fetched. +- **A document that is not text** - a Word, PowerPoint or Excel file, or a mail - is put into + `incoming/` as it is. `tools/wikitool raw derive incoming/` turns it into a folder with the + untouched original and a readable text beside it, with a header saying how it was made. The + folder is accepted whole; the agent runs this itself when it meets such a file. - **A page behind a paywall or a login** is saved from your browser into `incoming/`, HTML only. The agent derives the same text from that file with `raw fetch --html`. diff --git a/docs/reference/commands/README.md b/docs/reference/commands/README.md index 6ce128c..4aa0b82 100644 --- a/docs/reference/commands/README.md +++ b/docs/reference/commands/README.md @@ -79,6 +79,7 @@ Full records: [provenance.md](provenance.md) Full records: [raw-material-and-uploads.md](raw-material-and-uploads.md) - [`raw fetch`](raw-material-and-uploads.md#raw-fetch) - Capture a web page the user names into `incoming/`: the HTML as received plus a derived text, for `raw accept` to promote. +- [`raw derive`](raw-material-and-uploads.md#raw-derive) - Derive reading text from a PDF, e-mail, Office file or web page in `incoming/` and leave the original beside it, for `raw accept` to promote. - [`raw capture`](raw-material-and-uploads.md#raw-capture) - Capture documentation from a git repository into `incoming//`, with a manifest naming the repository, ref rule and commit, for `raw accept` to promote. - [`raw status`](raw-material-and-uploads.md#raw-status) - Report which captured bundles under `raw/` have fallen behind their repository, with each changed file as `A`/`M`/`D`. - [`raw pending`](raw-material-and-uploads.md#raw-pending) - List what waits in `incoming/`, oldest first, and name the entry an ingest without an argument takes next. diff --git a/docs/reference/commands/raw-material-and-uploads.md b/docs/reference/commands/raw-material-and-uploads.md index afaafbe..edcff22 100644 --- a/docs/reference/commands/raw-material-and-uploads.md +++ b/docs/reference/commands/raw-material-and-uploads.md @@ -6,6 +6,7 @@ Bringing new material into `raw/`: fetching and capturing it, seeing what is wai ## Contents - [`raw fetch`](#raw-fetch) +- [`raw derive`](#raw-derive) - [`raw capture`](#raw-capture) - [`raw status`](#raw-status) - [`raw pending`](#raw-pending) @@ -82,6 +83,66 @@ tools/wikitool raw fetch --html incoming/post.html --url https://example.org/blo - `wikitool raw accept` - promotes the written files into `raw/` - `instructions/wiki-ingest/SKILL.md` - where a URL to ingest starts +## `raw derive` + +Derive reading text from a PDF, e-mail, Office file or web page in `incoming/` and leave the original beside it, for `raw accept` to promote. + +**Synopsis** + +- `wikitool raw derive incoming/ [--name ]` - Move the file into the new folder `incoming//` as `original.` and write `incoming//derived.md` +- `wikitool raw derive incoming/` - Write a `.md` beside every file in the folder that has a filter + +**Options** + +```text +Arguments: + path A file or a folder directly in incoming/ [required] + +Options: + --name Folder stem for a single file instead of the file's own stem + -h, --help Show this message and exit. +``` + +**Examples** + +```bash +tools/wikitool raw derive incoming/report.docx +tools/wikitool raw derive incoming/Angebot-XYZ.xlsx --name angebot-xyz +tools/wikitool raw derive incoming/meeting-files +``` + +**Behaviour** + +- Writes into `incoming/` only, never into `raw/`, and reads no network: `raw accept` promotes the folder afterwards as one source under `raw////`. +- A single file directly in `incoming/` becomes a folder named after its stem (or `--name`) holding `original.` - the original bytes, moved - and `derived.md`. The fixed names keep the path inside the path budget; the header of `derived.md` records the file's own name as `original_name`. +- A folder is derived in place: each file with a filter gets `.md` beside it, nothing is renamed, and the files without a filter are named in the output. +- `derived.md` starts with a fixed header (`derived_by`, `filter`, `engine`, `derived`, `derived_from`, `original_name` for a renamed file, `title`, then the fields of the format) followed by the derived text. The same bytes always give the same text; only `derived` differs between runs. `engine` names the tool version and, where a package reads the format, its version. +- Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, `.xlsm`) and e-mail (`.eml`) read with the standard library only. A mail with attachments is refused. +- A name already taken under `raw/` is a WARN, not a refusal: whether the folder is a later edition of that source or a separate one is the human's call. A later edition of a derived source is replaced file by file with `raw accept --replaces`; there is no `--replaces-bundle` for it. +- After a success the output prints the warnings the filters raised, at most three `CHECK` lines per format that was derived in this run - what to look at before the text is trusted - and the `raw accept` line for the result. +- Never edit `derived.md` by hand: a better derivation is a new `raw derive` of the original. + +**Properties** + +- Changes files: yes +- Idempotent: no +- Atomic: Yes for what it leaves behind - everything is read, derived and checked before the first write; the files are created exclusively, and a failure part-way puts back what this call changed +- Counts toward the session call budget: yes +- Can reach the network: no + +**Exit codes** + +- `0` - success +- `1` - The path is not directly in `incoming/`, does not exist, or no filter reads the file's extension +- `1` - The target folder or a `.md` already exists, or the path would be over the path budget +- `1` - The document cannot be read (damaged, encrypted, a mail with attachments) or the filter's package is not installed + +**See also** + +- `raw/CONTRACT.md` "Getting a document in: `raw derive`" - the rules and why +- `wikitool raw accept` - promotes the folder into `raw/` +- `instructions/wiki-ingest/SKILL.md` - where a PDF, mail or Office file to ingest starts + ## `raw capture` Capture documentation from a git repository into `incoming//`, with a manifest naming the repository, ref rule and commit, for `raw accept` to promote. @@ -232,7 +293,8 @@ tools/wikitool raw pending --json - Order: oldest first by modification time. A bundle or folder counts as new as its newest file; a tie goes by name. The mtime is when a document last changed only if it was copied with its timestamps kept (`cp -p`, `rsync -a`, an unpacked archive) - for a download or a `raw fetch` it is merely when it was dropped. - Each candidate shows its path(s), kind, file count and mtime, and whether `raw accept` would take it as it stands - the same checks, minus `--fidelity`/`--authority`. One it would refuse is listed with the reason and skipped: it needs a human. - The default is the first candidate `raw accept` would take, and the output names it. -- `--json` prints the same candidates in the same order: `kind`, `paths`, `files`, `mtime`, `acceptable`, `reason`, `default`. +- `--json` prints the same candidates in the same order: `kind`, `paths`, `files`, `mtime`, `acceptable`, `reason`, `default`, `hint`. +- A lone file that a `raw derive` filter reads (a PDF, mail, Office file or web page) is listed with the `raw derive` line for it, in text output and as `hint` in JSON (`null` for every other candidate). The hint changes nothing about the order or the default. - Reads directory listings and `lstat` only, never a file's content; an empty `incoming/` is exit 0 with nothing to do. **Properties** diff --git a/instructions/wiki-ingest/SKILL.md b/instructions/wiki-ingest/SKILL.md index b9f26e0..51e3700 100644 --- a/instructions/wiki-ingest/SKILL.md +++ b/instructions/wiki-ingest/SKILL.md @@ -44,8 +44,9 @@ validator complains - and the ticked list is the only record that they happened. 1. **Read the source.** Read the file completely, wherever it currently sits - `incoming/` for the normal path, or already under `raw/` when the run started there (a file `capture-session` - just wrote, for instance, which skips step 5 entirely). If it is binary or an image, note its - presence and what it shows. + just wrote, for instance, which skips step 5 entirely). A PDF, mail, Office file or ZIP: run + `tools/wikitool raw derive ` first and read the `.md` - its output names what to check. + Any other binary or image: note its presence and what it shows. **The user named nothing** ("ingest", "process the inbox")? Pick the entry from the queue: @@ -250,7 +251,7 @@ validator complains - and the ticked list is the only record that they happened. incoming/ [incoming/ ...] ``` - List every file this one source produced (e.g. an uploaded PDF plus its converted Markdown) + List every file this one source produced (e.g. a `raw fetch` pair) in the same call, so they land bundled together rather than as two independent promotions. A file already in `raw/` skips this step entirely. A folder is accepted as a whole, on its own: @@ -461,7 +462,7 @@ validator complains - and the ticked list is the only record that they happened. ## wikitool commands used -`raw pending`, `raw fetch`, `raw accept`, `raw status`, `raw capture`, `search`, `types describe`, `task new`, `task list`, `task close`, +`raw pending`, `raw fetch`, `raw derive`, `raw accept`, `raw status`, `raw capture`, `search`, `types describe`, `task new`, `task list`, `task close`, `new project`, `new source`, `new entity`, `new concept`, `touch`, `cite add`, `xref add`, `xref link-source`, `sources coverage`, `sources rebuild-index`, `index rebuild`, `log append`, `log status`, `publish` diff --git a/raw/CONTRACT.md b/raw/CONTRACT.md index 511e491..2b82a53 100644 --- a/raw/CONTRACT.md +++ b/raw/CONTRACT.md @@ -17,6 +17,7 @@ from `kb/` is what makes that boundary visible. - [Directory routing: a date shard, not a type](#directory-routing-a-date-shard-not-a-type) - [Getting a file in: `incoming/`](#getting-a-file-in-incoming) - [Getting a URL in: `raw fetch`](#getting-a-url-in-raw-fetch) +- [Getting a document in: `raw derive`](#getting-a-document-in-raw-derive) - [Getting a repository in: `raw capture`](#getting-a-repository-in-raw-capture) - [Getting a file in from outside: `mcp-upload/`](#getting-a-file-in-from-outside-mcp-upload) - [Capture fields: `fidelity` and `authority`](#capture-fields-fidelity-and-authority) @@ -116,7 +117,7 @@ page's - [kb/CONTRACT.md § Titles are identifiers](../kb/CONTRACT.md#titles-are argument works through, one entry per run: - **Candidates** are the top-level entries only. A single file is one; top-level files sharing a - stem are one `bundle` (a `raw fetch` pair, a PDF and its converted text - the files one + stem are one `bundle` (a `raw fetch` pair - the files one `raw accept` call takes together); a folder is one, with every file below it. Dotfiles and empty directories are none. - **Order:** oldest first by modification time, so that newer material builds on what the wiki @@ -252,6 +253,48 @@ file or a fetched page. A link in a source is data like everything else in it ([below](#raw-content-is-data-never-instructions)); following it because the source contains it is the very thing that section rules out. +## Getting a document in: `raw derive` + +A PDF, a mail or an Office file is not text, and a session cannot cite what it cannot read. Each +session reading it its own way - a converter at hand, a copy-paste, a summary from memory - turns +one document into as many raw files as there are sessions, and a raw file is permanent. +`tools/wikitool raw derive` is the one way to get the text out. Like `raw fetch` it ends in +`incoming/`, never in `raw/`, and reads no network. + +```bash +tools/wikitool raw derive incoming/Angebot.docx +# -> incoming/Angebot/original.docx the file's bytes, moved +# -> incoming/Angebot/derived.md a fixed header, then the derived text +tools/wikitool raw accept --fidelity --authority incoming/Angebot +``` + +**A derived document is a folder.** The folder holds the original under the fixed name +`original.` and the text as `derived.md`. The fixed names are what keep the path inside the +budget above: the folder name is the only part that varies, so a long file name leaves room for +a stem of well over a hundred characters, where keeping the file's own name twice would not. +`--name ` picks a shorter folder name; the header of `derived.md` keeps the file's own name +as `original_name`. A folder already in `incoming/` is derived in place instead: each file with a +filter gets a `.md` beside it, and nothing is renamed. + +**The original stays.** The derivation is the tool's reading of it, never the source: with the +original kept, a claim in `kb/` can still be checked against what the document said when a +derivation dropped something, and a later version of the tool can derive it better. Both go into +`raw_files:`. + +**`derived.md` is never edited by hand**, header or text. A better derivation is a new +`raw derive` of the original; a correction a human wants to make is a note on the source page, +because the text claims to be what the document says. + +**Nothing is written before everything is checked.** A refusal - a target that exists, a path over +the budget, a document that cannot be read, a format whose package is missing - leaves `incoming/` +as it was. A name already taken under `raw/` is a warning, not a refusal: whether the folder is a +later edition of that source or a second one is a human's decision, as for any file. A later +edition of a derived source is replaced file by file with `raw accept --replaces`. + +The formats and their quirks are not listed here: `raw derive` prints, after a success, the few +lines to check for the formats it derived in that run, and `wikitool raw derive -h` names the +formats it reads. + ## Getting a repository in: `raw capture` Documentation that lives in a git repository - a service's `docs/`, a project's `README.md` - is diff --git a/tools/CONTRACT.md b/tools/CONTRACT.md index 04060c3..8e205cc 100644 --- a/tools/CONTRACT.md +++ b/tools/CONTRACT.md @@ -123,6 +123,7 @@ sources coverage read idempotent budget:counted exit:0 sources trace read idempotent budget:counted exit:0,1 Trace provenance in either direction: raw file, or page. sources rebuild-index write idempotent budget:counted exit:0,1 Regenerate the `kb/provenance.md` reverse index. raw fetch write non-idempotent budget:counted exit:0,1 Capture a web page the user names into `incoming/`: the HTML as received plus a derived text, for `raw accept` to promote. +raw derive write non-idempotent budget:counted exit:0,1 Derive reading text from a PDF, e-mail, Office file or web page in `incoming/` and leave the original beside it, for `raw accept` to promote. raw capture write non-idempotent budget:counted exit:0,1 Capture documentation from a git repository into `incoming//`, with a manifest naming the repository, ref rule and commit, for `raw accept` to promote. raw status read idempotent budget:counted exit:0 Report which captured bundles under `raw/` have fallen behind their repository, with each changed file as `A`/`M`/`D`. raw pending read idempotent budget:counted exit:0 List what waits in `incoming/`, oldest first, and name the entry an ingest without an argument takes next. @@ -1466,6 +1467,64 @@ Capture a web page the user names into `incoming/`: the HTML as received plus a - `wikitool raw accept` - promotes the written files into `raw/` - `instructions/wiki-ingest/SKILL.md` - where a URL to ingest starts +#### `raw derive` + +Derive reading text from a PDF, e-mail, Office file or web page in `incoming/` and leave the original beside it, for `raw accept` to promote. + +**SYNOPSIS** + +- `wikitool raw derive incoming/ [--name ]` - Move the file into the new folder `incoming//` as `original.` and write `incoming//derived.md` +- `wikitool raw derive incoming/` - Write a `.md` beside every file in the folder that has a filter + +**PROPERTIES** + +- effect: write +- idempotent: no +- atomic: Yes for what it leaves behind - everything is read, derived and checked before the first write; the files are created exclusively, and a failure part-way puts back what this call changed +- budget: counted +- network: no + +**EXAMPLES** + +- `tools/wikitool raw derive incoming/report.docx` +- `tools/wikitool raw derive incoming/Angebot-XYZ.xlsx --name angebot-xyz` +- `tools/wikitool raw derive incoming/meeting-files` + +**EXIT STATUS** + +- 0 success +- 1 The path is not directly in `incoming/`, does not exist, or no filter reads the file's extension +- 1 The target folder or a `.md` already exists, or the path would be over the path budget +- 1 The document cannot be read (damaged, encrypted, a mail with attachments) or the filter's package is not installed + +**ON FAILURE** + +- The path is not directly in `incoming/`, does not exist, or no filter reads the file's extension -> Nothing was written. Fix the argument and retry once - a file in a subfolder is moved up, or its folder is derived whole +- The target folder or a `.md` already exists, or the path would be over the path budget -> Nothing was written. Remove what is there or pass `--name `, then retry once. An existing `.md` is never overwritten +- The document cannot be read (damaged, encrypted, a mail with attachments) or the filter's package is not installed -> Nothing was written. A missing package is reported with the preflight hint - show it to the user and stop, never install it. Any other filter keeps working + +**NEVER** + +- Never edit `derived.md` or its header by hand. +- Never derive a file the user did not put into `incoming/` for this purpose. + +**NOTES** + +- Writes into `incoming/` only, never into `raw/`, and reads no network: `raw accept` promotes the folder afterwards as one source under `raw////`. +- A single file directly in `incoming/` becomes a folder named after its stem (or `--name`) holding `original.` - the original bytes, moved - and `derived.md`. The fixed names keep the path inside the path budget; the header of `derived.md` records the file's own name as `original_name`. +- A folder is derived in place: each file with a filter gets `.md` beside it, nothing is renamed, and the files without a filter are named in the output. +- `derived.md` starts with a fixed header (`derived_by`, `filter`, `engine`, `derived`, `derived_from`, `original_name` for a renamed file, `title`, then the fields of the format) followed by the derived text. The same bytes always give the same text; only `derived` differs between runs. `engine` names the tool version and, where a package reads the format, its version. +- Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, `.xlsm`) and e-mail (`.eml`) read with the standard library only. A mail with attachments is refused. +- A name already taken under `raw/` is a WARN, not a refusal: whether the folder is a later edition of that source or a separate one is the human's call. A later edition of a derived source is replaced file by file with `raw accept --replaces`; there is no `--replaces-bundle` for it. +- After a success the output prints the warnings the filters raised, at most three `CHECK` lines per format that was derived in this run - what to look at before the text is trusted - and the `raw accept` line for the result. +- Never edit `derived.md` by hand: a better derivation is a new `raw derive` of the original. + +**SEE ALSO** + +- `raw/CONTRACT.md` "Getting a document in: `raw derive`" - the rules and why +- `wikitool raw accept` - promotes the folder into `raw/` +- `instructions/wiki-ingest/SKILL.md` - where a PDF, mail or Office file to ingest starts + #### `raw capture` Capture documentation from a git repository into `incoming//`, with a manifest naming the repository, ref rule and commit, for `raw accept` to promote. @@ -1611,7 +1670,8 @@ List what waits in `incoming/`, oldest first, and name the entry an ingest witho - Order: oldest first by modification time. A bundle or folder counts as new as its newest file; a tie goes by name. The mtime is when a document last changed only if it was copied with its timestamps kept (`cp -p`, `rsync -a`, an unpacked archive) - for a download or a `raw fetch` it is merely when it was dropped. - Each candidate shows its path(s), kind, file count and mtime, and whether `raw accept` would take it as it stands - the same checks, minus `--fidelity`/`--authority`. One it would refuse is listed with the reason and skipped: it needs a human. - The default is the first candidate `raw accept` would take, and the output names it. -- `--json` prints the same candidates in the same order: `kind`, `paths`, `files`, `mtime`, `acceptable`, `reason`, `default`. +- `--json` prints the same candidates in the same order: `kind`, `paths`, `files`, `mtime`, `acceptable`, `reason`, `default`, `hint`. +- A lone file that a `raw derive` filter reads (a PDF, mail, Office file or web page) is listed with the `raw derive` line for it, in text output and as `hint` in JSON (`null` for every other candidate). The hint changes nothing about the order or the default. - Reads directory listings and `lstat` only, never a file's content; an empty `incoming/` is exit 0 with nothing to do. **SEE ALSO** diff --git a/tools/README.md b/tools/README.md index 9c1d28a..446391d 100644 --- a/tools/README.md +++ b/tools/README.md @@ -112,6 +112,7 @@ tools/ kb_state.py the KB version (.wikitool-kb.json) and the migration chain corpus_diff.py invariant comparison of kb/ between two revisions web_capture.py `raw fetch`'s core: fetch a page, decide its charset, derive Markdown-like text from the HTML - standard library only, deterministic + filters/ the import filters behind `raw derive`: a registry (`__init__.py`) and one module per format family (HTML, mail, Office), each registering the extensions it reads - standard library, or a package imported lazily so a missing one fails only its own format repo_capture.py `raw capture`/`raw status`'s core: resolve a ref rule, fetch it into a bare cache, select files by glob as blobs, the `_capture.json` manifest - git with the host's credentials, never a prompt; plus the one write back, a single-file commit pushed without force for `export guidelines` guideline_export.py `export guidelines`' core: select pages by `search` predicates, render them into one deterministic GUIDELINES.md, and recognise an export again (`EXPORT_MARKER`) - with no CLI attached search/ pluggable search backends, plus service.py - the search core diff --git a/tools/chemenu/cli_contract.py b/tools/chemenu/cli_contract.py index 994fe28..20fcdc9 100644 --- a/tools/chemenu/cli_contract.py +++ b/tools/chemenu/cli_contract.py @@ -287,7 +287,7 @@ GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = ( "sources coverage", "sources trace", "sources rebuild-index", )), ("Raw material and uploads", ( - "raw fetch", "raw capture", "raw status", "raw pending", "raw accept", + "raw fetch", "raw derive", "raw capture", "raw status", "raw pending", "raw accept", "upload list", "upload show", "upload accept", "upload reject", )), ("Git", ( diff --git a/tools/chemenu/commands/raw_cmd.py b/tools/chemenu/commands/raw_cmd.py index a761230..c24da8c 100644 --- a/tools/chemenu/commands/raw_cmd.py +++ b/tools/chemenu/commands/raw_cmd.py @@ -25,6 +25,12 @@ sit directly in `incoming/`: today's shard, so a bundle never mixes an old capture date with today's (Gitea #67 decision, "Datums-Shard" § "Bündelort"). +`raw derive` is the way in for a document that is not text (PDF, mail, Office +file): it turns one file in `incoming/` into a folder `/` holding the +untouched `original.` and a `derived.md` made by the registered filter +(`chemenu.filters`), so `raw accept` takes the folder as one source. Everything +is checked before the first write; the original moves in last. + `raw pending` reads the same queue without changing it: the candidates in `incoming/` oldest first, each judged by the very checks `raw accept` runs before it moves anything (`_Refused` is how those checks report without @@ -87,7 +93,7 @@ from typing import Annotated, Optional import typer from rich.markup import escape -from chemenu import cli_contract, config, guideline_export, repo_capture, web_capture +from chemenu import cli_contract, config, filters, guideline_export, repo_capture, web_capture from chemenu.commands._util import fail, path_budget_problem_for, rel_path, success from chemenu.errors import BackendError, ChemenuError, ValidationError from chemenu.frontmatter_io import write_page @@ -962,6 +968,20 @@ def _replace_bundle( "the list above - they are still untracked.") +def _warn_underived(resolved: list[Path]) -> None: + """A file `raw derive` could read, promoted with no `.md` beside it, leaves + the source without a text the wiki can read - say so, and carry on.""" + stems_with_text = {p.stem for p in resolved if p.suffix.lower() == ".md"} + for path in resolved: + hint = _derive_hint(path) + if hint is not None and path.stem not in stems_with_text: + typer.echo( + f"WARN {rel_path(path)} has a text filter and no derived .md beside it - the " + f"source would have no text to read. Run {hint} first and accept the folder it " + "makes, unless the file is meant to be kept as it is." + ) + + @cli_contract.record(cli_contract.CommandRecord( path="raw accept", summary="Promote one or more files, or one folder, from `incoming/` into `raw/`.", @@ -1264,6 +1284,7 @@ def raw_accept_command( return _refusals_fail(_check_files, resolved) + _warn_underived(resolved) pages = None target_page = None @@ -1378,6 +1399,7 @@ class _Candidate: files: int mtime_ns: int reason: Optional[str] # None when `raw accept` would take it as it stands + hint: Optional[str] = None # the `raw derive` line for a lone file a filter reads def label(self) -> str: return ", ".join(rel_path(p) + ("/" if self.kind == "folder" else "") for p in self.paths) @@ -1449,6 +1471,7 @@ def _pending_candidates() -> list[_Candidate]: len(paths), max(p.lstat().st_mtime_ns for p in paths), _refusal(_plain_files, paths), + _derive_hint(paths[0]) if len(paths) == 1 else None, )) candidates.sort(key=lambda c: (c.mtime_ns, os.fsencode(c.paths[0].name))) return candidates @@ -1480,7 +1503,11 @@ def _pending_candidates() -> list[_Candidate]: "human.", "The default is the first candidate `raw accept` would take, and the output names it.", "`--json` prints the same candidates in the same order: `kind`, `paths`, `files`, " - "`mtime`, `acceptable`, `reason`, `default`.", + "`mtime`, `acceptable`, `reason`, `default`, `hint`.", + "A lone file that a `raw derive` filter reads (a PDF, mail, Office file or web page) is " + "listed with the `raw derive` line for it, in text output and as `hint` in JSON " + "(`null` for every other candidate). The hint changes nothing about the order or the " + "default.", "Reads directory listings and `lstat` only, never a file's content; an empty " "`incoming/` is exit 0 with nothing to do.", ), @@ -1514,6 +1541,7 @@ def raw_pending_command( "acceptable": c.reason is None, "reason": c.reason, "default": c is default, + "hint": c.hint, } for c in candidates ], indent=2, ensure_ascii=False)) @@ -1531,6 +1559,8 @@ def raw_pending_command( typer.echo(f" not acceptable: {first}") for line in rest: typer.echo(f" {line}") + if c.hint is not None: + typer.echo(f" text: {c.hint} (derives readable text, keeps the original)") if default is None: typer.echo("No candidate can be accepted as it stands - each needs a human (reasons above).") return @@ -1800,6 +1830,345 @@ def raw_fetch_command( )) +# --- raw derive --------------------------------------------------------------- +# +# The sanctioned intake for a document that is not text: a PDF, a mail, an +# Office file. Like `raw fetch` it ends in `incoming/`, never in `raw/`. The +# formats live in `chemenu.filters`; this adapter decides names and folders and +# what the caller is told. Everything is read, derived and checked before the +# first write, so a refusal leaves `incoming/` as it was. + +DERIVE_MAX_BYTES = 100 * 1024 * 1024 +ORIGINAL_STEM = "original" +DERIVED_NAME = "derived.md" + + +@dataclass(frozen=True) +class _Derived: + """One finished derivation, not yet written.""" + + filter: filters.Filter + derivation: filters.Derivation + source: Path # the file in incoming/ + markdown: bytes + target: Path # where the .md goes + + +def _derive_hint(path: Path) -> Optional[str]: + """The `raw derive` line for a file a filter reads, else None.""" + if filters.filter_for(path.name) is None: + return None + return f"tools/wikitool raw derive {rel_path(path)}" + + +def _read_for_derive(path: Path) -> bytes: + size = path.stat().st_size + if size > DERIVE_MAX_BYTES: + raise _Refused( + f"{rel_path(path)} is {size // (1024 * 1024)} MiB; raw derive reads at most " + f"{DERIVE_MAX_BYTES // (1024 * 1024)} MiB." + ) + return path.read_bytes() + + +def _run_filter(candidate: filters.Filter, data: bytes, path: Path) -> filters.Derivation: + try: + filters.require_available(candidate) + return candidate.derive(data, path.name) + except ValidationError as exc: + raise _Refused(f"{rel_path(path)}: {exc}") from None + + +def _collision_warning(name: str) -> Optional[str]: + holder = _occupied_stems(config.RAW_DIR).get(name) + if holder is None: + return None + return _folder_collision_message(name, holder) + + +def _plan_derive_file(path: Path, name: Optional[str]): + """The single-file form: `incoming//` with the original moved in as + `original.` and the derived text beside it.""" + incoming = _incoming_dir() + if not path.is_file(): + raise _Refused(f"{rel_path(path)} does not exist or is not a file.") + _check_directly_in_incoming(path, incoming) + if path.name.startswith(".") or path.is_symlink(): + raise _Refused(f"{rel_path(path)} is hidden or a symlink - raw derive does not take it.") + candidate = filters.filter_for(path.name) + if candidate is None: + listed = ", ".join(filters.supported_extensions()) + raise _Refused( + f"No filter reads {filters.extension_of(path.name) or 'a file without an extension'} " + f"({rel_path(path)}). raw derive reads: {listed}. Any other file is accepted as it is " + "- note what it shows." + ) + stem = name if name is not None else path.stem + if _unusable_name(stem): + raise _Refused( + f"{stem!r} is not a usable folder name: one directory name - no path separator, not " + f"hidden, not {repo_capture.MANIFEST_NAME}. Pass --name ." + ) + folder = incoming / stem + if os.path.lexists(folder): + raise _Refused( + f"{rel_path(folder)} already exists in incoming/ - raw derive never overwrites. Accept " + "or remove what is there, or pass --name ." + ) + extension = filters.extension_of(path.name) + original = folder / f"{ORIGINAL_STEM}{extension}" + derived = folder / DERIVED_NAME + for target in (original, derived): + problem = path_budget_problem_for(_shard_dir() / stem / target.name) + if problem: + raise _Refused( + f"Cannot write {problem}. The folder name comes from the file's name: pass " + "--name ." + ) + derivation = _run_filter(candidate, _read_for_derive(path), path) + markdown = filters.render( + candidate, + derivation, + derived_from=original.name, + original_name=path.name, + ) + return folder, original, derived, candidate, derivation, markdown, stem + + +def _move_in(source: Path, folder: Path, original: Path, derived: Path, markdown: bytes) -> None: + """Create `folder`, write the derived text, then move the original in - last, + so a failure before it has taken nothing from `incoming/`. A failure part-way + puts back what this call changed.""" + moved = False + try: + folder.mkdir() + with derived.open("xb") as handle: + handle.write(markdown) + source.rename(original) + moved = True + except OSError as exc: + if moved: + original.rename(source) + derived.unlink(missing_ok=True) + if folder.is_dir(): + try: + folder.rmdir() + except OSError: + pass + fail(escape(f"Could not write {rel_path(folder)}: {exc}. incoming/ was left as it was.")) + + +def _plan_derive_folder(folder: Path): + """The folder form: a `.md` beside every file that has a filter.""" + incoming = _incoming_dir() + try: + rel = folder.relative_to(incoming) + except ValueError: + raise _Refused( + f"{rel_path(folder)} is not under incoming/ - raw derive only takes material from there." + ) from None + if len(rel.parts) != 1: + raise _Refused( + f"incoming/{rel.as_posix()} is not directly in incoming/ - a folder is derived from " + f"the top: raw derive incoming/{rel.parts[0]}" + ) + if folder.name.startswith(".") or folder.is_symlink(): + raise _Refused(f"{rel_path(folder)} is hidden or a symlink - raw derive does not take it.") + if (folder / repo_capture.MANIFEST_NAME).is_file(): + raise _Refused(f"{rel_path(folder)}/ is a captured bundle; a captured bundle is not derived.") + files, _dirs, refused = _walk_folder(folder) + if refused: + listed = "\n".join(f" - {entry}" for entry in refused) + raise _Refused( + f"{rel_path(folder)}/ holds entries raw accept would not take - hidden entries, " + f"symlinks and special files:\n{listed}\n Remove or replace them in incoming/, then " + "derive the folder again." + ) + plan: list[tuple[Path, filters.Filter, Path]] = [] + untouched: list[Path] = [] + for file in files: + candidate = filters.filter_for(file.name) + if candidate is None: + untouched.append(file) + else: + plan.append((file, candidate, file.with_suffix(".md"))) + if not plan: + raise _Refused( + f"No file in {rel_path(folder)}/ has a filter. raw derive reads: " + f"{', '.join(filters.supported_extensions())}." + ) + targets: dict[Path, Path] = {} + for file, _candidate, md in plan: + if md in targets: + raise _Refused( + f"{rel_path(targets[md])} and {rel_path(file)} would both derive to {rel_path(md)}. " + "Rename one of them in incoming/ and derive the folder again." + ) + targets[md] = file + if os.path.lexists(md): + raise _Refused( + f"{rel_path(md)} already exists - raw derive never overwrites. Remove it or " + "rename it, then derive the folder again." + ) + problem = path_budget_problem_for(_shard_dir() / folder.name / md.relative_to(folder)) + if problem: + raise _Refused( + f"Cannot write {problem}. The path comes from the folder in incoming/: shorten the " + "folder's name or the names inside it." + ) + derived: list[_Derived] = [] + for file, candidate, md in plan: + derivation = _run_filter(candidate, _read_for_derive(file), file) + markdown = filters.render(candidate, derivation, derived_from=file.name) + derived.append(_Derived(candidate, derivation, file, markdown, md)) + return derived, untouched + + +def _echo_notes(derivations: list[filters.Derivation]) -> None: + for derivation in derivations: + for warning in derivation.warnings: + typer.echo(f"WARN {warning}") + for line in filters.unique(c for d in derivations for c in d.checks): + typer.echo(f"CHECK {line}") + + +def _derive_next(target: Path) -> str: + return ( + " Next (after the commitment question in wiki-ingest):\n" + f" tools/wikitool raw accept --fidelity --authority {rel_path(target)}" + ) + + +@cli_contract.record(cli_contract.CommandRecord( + path="raw derive", + summary="Derive reading text from a PDF, e-mail, Office file or web page in `incoming/` and " + "leave the original beside it, for `raw accept` to promote.", + synopsis=( + cli_contract.Variant( + usage="raw derive incoming/ [--name ]", + notes="Move the file into the new folder `incoming//` as `original.` and " + "write `incoming//derived.md`", + ), + cli_contract.Variant( + usage="raw derive incoming/", + notes="Write a `.md` beside every file in the folder that has a filter", + ), + ), + properties=cli_contract.Properties( + effect=cli_contract.Effect.WRITE, + idempotent=cli_contract.Idempotent.NO, + atomic="Yes for what it leaves behind - everything is read, derived and checked before " + "the first write; the files are created exclusively, and a failure part-way puts back " + "what this call changed", + budget=cli_contract.Budget.COUNTED, + network=cli_contract.Network.NO, + ), + notes=( + "Writes into `incoming/` only, never into `raw/`, and reads no network: `raw accept` " + "promotes the folder afterwards as one source under `raw////`.", + "A single file directly in `incoming/` becomes a folder named after its stem (or " + "`--name`) holding `original.` - the original bytes, moved - and `derived.md`. The " + "fixed names keep the path inside the path budget; the header of `derived.md` records " + "the file's own name as `original_name`.", + "A folder is derived in place: each file with a filter gets `.md` beside it, " + "nothing is renamed, and the files without a filter are named in the output.", + "`derived.md` starts with a fixed header (`derived_by`, `filter`, `engine`, `derived`, " + "`derived_from`, `original_name` for a renamed file, `title`, then the fields of the " + "format) followed by the derived text. The same bytes always give the same text; only " + "`derived` differs between runs. `engine` names the tool version and, where a package " + "reads the format, its version.", + "Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, " + "`.xlsm`) and e-mail (`.eml`) read with the standard library only. A mail with " + "attachments is refused.", + "A name already taken under `raw/` is a WARN, not a refusal: whether the folder is a " + "later edition of that source or a separate one is the human's call. A later edition of " + "a derived source is replaced file by file with `raw accept --replaces`; there is no " + "`--replaces-bundle` for it.", + "After a success the output prints the warnings the filters raised, at most three " + "`CHECK` lines per format that was derived in this run - what to look at before the " + "text is trusted - and the `raw accept` line for the result.", + "Never edit `derived.md` by hand: a better derivation is a new `raw derive` of the original.", + ), + failures=( + cli_contract.Failure( + cause="The path is not directly in `incoming/`, does not exist, or no filter reads " + "the file's extension", + reaction="Nothing was written. Fix the argument and retry once - a file in a " + "subfolder is moved up, or its folder is derived whole", + ), + cli_contract.Failure( + cause="The target folder or a `.md` already exists, or the path would be over the " + "path budget", + reaction="Nothing was written. Remove what is there or pass `--name `, " + "then retry once. An existing `.md` is never overwritten", + ), + cli_contract.Failure( + cause="The document cannot be read (damaged, encrypted, a mail with attachments) or " + "the filter's package is not installed", + reaction="Nothing was written. A missing package is reported with the preflight " + "hint - show it to the user and stop, never install it. Any other filter keeps " + "working", + ), + ), + examples=( + "tools/wikitool raw derive incoming/report.docx", + "tools/wikitool raw derive incoming/Angebot-XYZ.xlsx --name angebot-xyz", + "tools/wikitool raw derive incoming/meeting-files", + ), + never=( + "Never edit `derived.md` or its header by hand.", + "Never derive a file the user did not put into `incoming/` for this purpose.", + ), + see_also=( + "`raw/CONTRACT.md` \"Getting a document in: `raw derive`\" - the rules and why", + "`wikitool raw accept` - promotes the folder into `raw/`", + "`instructions/wiki-ingest/SKILL.md` - where a PDF, mail or Office file to ingest starts", + ), +)) +@app.command("derive") +def raw_derive_command( + path: Path = typer.Argument(..., help="A file or a folder directly in incoming/"), + name: Annotated[Optional[str], typer.Option( + "--name", help="Folder stem for a single file instead of the file's own stem" + )] = None, +): + """Derive reading text from a document in incoming/ and keep the original. + See raw/CONTRACT.md "Getting a document in: `raw derive`".""" + resolved = _resolve(path) + if resolved.is_dir(): + if name is not None: + fail("--name applies to a single file; a folder keeps its own name.") + derived, untouched = _refusals_fail(_plan_derive_folder, resolved) + _write_exclusive([(d.target, d.markdown) for d in derived]) + lines = [f"Derived {len(derived)} file(s) in {rel_path(resolved)}/:"] + lines += [f" {rel_path(d.source)} -> {rel_path(d.target)} ({d.filter.name})" for d in derived] + if untouched: + lines.append( + " No filter for, left as they are: " + ", ".join(rel_path(p) for p in untouched) + ) + success(escape("\n".join(lines))) + warning = _collision_warning(resolved.name) + if warning: + typer.echo("WARN " + warning) + _echo_notes([d.derivation for d in derived]) + typer.echo(_derive_next(resolved)) + return + + folder, original, derived_md, candidate, derivation, markdown, stem = _refusals_fail( + _plan_derive_file, resolved, name + ) + _move_in(resolved, folder, original, derived_md, markdown) + success(escape( + f"Derived {rel_path(derived_md)} from {resolved.name} ({candidate.name}); the original is " + f"now {rel_path(original)}." + )) + warning = _collision_warning(stem) + if warning: + typer.echo("WARN " + warning) + _echo_notes([derivation]) + typer.echo(_derive_next(folder)) + + # --- raw capture / raw status ------------------------------------------------- # # Documentation from a git repository, as one bundle per repository and ref @@ -1809,15 +2178,21 @@ def raw_fetch_command( # `raw/`: promoting stays `raw accept`'s job. -def _check_bundle_name(name: str) -> None: - if ( +def _unusable_name(name: str) -> bool: + """Not one plain directory name: empty, padded, hidden, a path, a control + character, or the manifest's reserved name.""" + return ( not name.strip() or name != name.strip() or name.startswith(".") or name == repo_capture.MANIFEST_NAME or any(c in name for c in "/\\:") or any(ord(c) < 0x20 for c in name) - ): + ) + + +def _check_bundle_name(name: str) -> None: + if _unusable_name(name): fail(escape( f"--name {name!r} is not a usable bundle name: one directory name - no path separator, " f"not hidden, not {repo_capture.MANIFEST_NAME}." diff --git a/tools/chemenu/filters/__init__.py b/tools/chemenu/filters/__init__.py new file mode 100644 index 0000000..9ba6710 --- /dev/null +++ b/tools/chemenu/filters/__init__.py @@ -0,0 +1,172 @@ +"""The import filters behind `wikitool raw derive`: one module per format family, +each registering the extensions it reads. + +A filter turns the bytes of a document into reading text plus the header fields +that describe it. The registry is code in this stack - a new format is a module +here, a registration and tests; the command, the header and the skill stay as +they are. The core knows neither `typer` nor `rich` and raises `ValidationError`. + +A filter's third-party package (if any) is imported inside the filter, never at +module level: a checkout whose environment predates the package fails only the +derivation of that one format, and every other command keeps running. +""" +from __future__ import annotations + +import importlib +import importlib.metadata +from dataclasses import dataclass +from typing import Iterable, Optional + +from chemenu import web_capture +from chemenu.errors import ValidationError +from chemenu.version import VersionError, read_version + +MAX_CHECKS = 3 + +PREFLIGHT_HINT = ( + "Run the preflight to install it (tools/preflight.sh, or tools/preflight.ps1 on Windows; " + "instructions/preflight.md). Every other format keeps working." +) + + +@dataclass(frozen=True) +class Derivation: + """What a filter makes of one document. + + `fields` are the format-specific header lines, after `title`; a value is a + string or a list of strings. `checks` are the short English lines `raw derive` + prints after a success - already narrowed to the case that occurred. + """ + + text: str + title: str = "" + fields: tuple[tuple[str, "str | list[str]"], ...] = () + warnings: tuple[str, ...] = () + checks: tuple[str, ...] = () + + def __post_init__(self) -> None: + if len(self.checks) > MAX_CHECKS: + raise ValueError(f"a filter prints at most {MAX_CHECKS} CHECK lines, got {len(self.checks)}") + + +class Filter: + """One format family. Subclasses set the class attributes and implement `derive`.""" + + name: str = "" + extensions: tuple[str, ...] = () + requires: tuple[str, ...] = () + + def available(self) -> bool: + for package in self.requires: + try: + importlib.import_module(package) + except ImportError: + return False + return True + + def missing(self) -> list[str]: + missing = [] + for package in self.requires: + try: + importlib.import_module(package) + except ImportError: + missing.append(package) + return missing + + def engine(self) -> str: + try: + parts = [f"wikitool {read_version()}"] + except VersionError: + parts = ["wikitool unknown"] + for package in self.requires: + try: + parts.append(f"{package} {importlib.metadata.version(package)}") + except importlib.metadata.PackageNotFoundError: + parts.append(package) + return " + ".join(parts) + + def derive(self, data: bytes, name: str) -> Derivation: + raise NotImplementedError + + +_REGISTRY: list[Filter] = [] + + +def register(candidate: Filter) -> Filter: + taken = {ext: f.name for f in _REGISTRY for ext in f.extensions} + for ext in candidate.extensions: + if ext in taken: + raise ValueError(f"extension {ext} is already read by the {taken[ext]} filter") + _REGISTRY.append(candidate) + return candidate + + +def all_filters() -> list[Filter]: + return list(_REGISTRY) + + +def extension_of(filename: str) -> str: + """The lowercase extension with its dot, or `""` for a name without one + (a leading dot alone is a hidden file, not an extension).""" + stripped = filename.lstrip(".") + return "." + stripped.rsplit(".", 1)[1].lower() if "." in stripped else "" + + +def filter_for(filename: str) -> Optional[Filter]: + ext = extension_of(filename) + for candidate in _REGISTRY: + if ext in candidate.extensions: + return candidate + return None + + +def supported_extensions() -> list[str]: + return sorted(ext for f in _REGISTRY for ext in f.extensions) + + +def require_available(candidate: Filter) -> None: + missing = candidate.missing() + if missing: + raise ValidationError( + f"The {candidate.name} filter needs the package {', '.join(missing)}, which this " + f"checkout's tool environment does not have. {PREFLIGHT_HINT}" + ) + + +def render( + candidate: Filter, + derivation: Derivation, + *, + derived_from: str, + original_name: Optional[str] = None, + derived_at: Optional[str] = None, +) -> bytes: + """The `.md` for one document: the fixed header, then the derived text. + `original_name` is left out for a file that was not renamed.""" + fields: list[tuple[str, "str | list[str]"]] = [ + ("derived_by", "wikitool raw derive"), + ("filter", candidate.name), + ("engine", candidate.engine()), + ("derived", derived_at or web_capture.utc_now()), + ("derived_from", derived_from), + ] + if original_name is not None: + fields.append(("original_name", original_name)) + fields.append(("title", derivation.title)) + fields.extend(derivation.fields) + text = derivation.text.strip("\n") + content = web_capture.header(fields) + "\n" + (text + "\n" if text else "") + return content.encode("utf-8") + + +def unique(lines: Iterable[str]) -> list[str]: + seen: set[str] = set() + out = [] + for line in lines: + if line not in seen: + seen.add(line) + out.append(line) + return out + + +from chemenu.filters import html_page, mail, ooxml # noqa: E402,F401 (register themselves) diff --git a/tools/chemenu/filters/html_page.py b/tools/chemenu/filters/html_page.py new file mode 100644 index 0000000..6434855 --- /dev/null +++ b/tools/chemenu/filters/html_page.py @@ -0,0 +1,34 @@ +"""HTML pages: an adapter on `web_capture` for a page that arrives without a URL.""" +from __future__ import annotations + +from chemenu import web_capture +from chemenu.filters import Derivation, Filter, register + + +class HtmlPageFilter(Filter): + name = "html" + extensions = (".html", ".htm", ".xhtml") + + def derive(self, data: bytes, name: str) -> Derivation: + decoded = web_capture.decode(data, None) + text = web_capture.derive_text(decoded.text, "") + warnings = [] + if decoded.replaced: + warnings.append( + f"{decoded.replaced} undecodable byte(s) under {decoded.charset} were replaced " + "with U+FFFD." + ) + if len(text) < web_capture.SHORT_TEXT_CHARS: + warnings.append( + f"The derived text is only {len(text)} characters - likely a script-rendered " + "page or an empty document." + ) + return Derivation( + text=text, + title=web_capture.title_of(web_capture.parse(decoded.text)), + fields=(("charset", f"{decoded.charset} (from: {decoded.source})"),), + warnings=tuple(warnings), + ) + + +register(HtmlPageFilter()) diff --git a/tools/chemenu/filters/mail.py b/tools/chemenu/filters/mail.py new file mode 100644 index 0000000..cc11116 --- /dev/null +++ b/tools/chemenu/filters/mail.py @@ -0,0 +1,177 @@ +"""E-mail: `.eml` (RFC 5322 / MIME), read with the standard library.""" +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from email import message_from_bytes, policy +from email.message import EmailMessage +from typing import Optional + +from chemenu import web_capture +from chemenu.errors import ValidationError +from chemenu.filters import Derivation, Filter, register + +_QUOTED = re.compile( + r"^(>|-{2,}\s*(Original Message|Ursprüngliche Nachricht)\s*-{2,}|On .{5,200} wrote:\s*$|Am .{5,200} schrieb .{0,200}:\s*$)", + re.IGNORECASE | re.MULTILINE, +) + + +@dataclass(frozen=True) +class Attachment: + name: str + data: bytes + content_type: str + + @property + def is_message(self) -> bool: + return self.content_type == "message/rfc822" or self.name.lower().endswith((".eml", ".msg")) + + +@dataclass +class ParsedMail: + headers: dict[str, "str | list[str]"] = field(default_factory=dict) + body: str = "" + warnings: list[str] = field(default_factory=list) + attachments: list[Attachment] = field(default_factory=list) + inline_images: int = 0 + + +def _header(msg: EmailMessage, name: str) -> str: + try: + return re.sub(r"\s+", " ", str(msg[name] or "")).strip() + except Exception: # a malformed header must not stop the derivation + raw = msg.get(name) + return re.sub(r"\s+", " ", str(raw or "")).strip() if raw is not None else "" + + +def _addresses(msg: EmailMessage, name: str) -> list[str]: + value = msg[name] + if value is None: + return [] + try: + found = [str(a).strip() for a in value.addresses] + if found: + return found + except Exception: + pass + text = _header(msg, name) + return [text] if text else [] + + +def _date(msg: EmailMessage) -> str: + value = msg["date"] + if value is None: + return "" + try: + return value.datetime.isoformat() + except Exception: + return _header(msg, "date") + + +def _text_of(part: EmailMessage) -> str: + try: + return part.get_content() + except (LookupError, UnicodeDecodeError, AssertionError): + payload = part.get_payload(decode=True) or b"" + return payload.decode(part.get_content_charset() or "utf-8", errors="replace") + + +def _filename(part: EmailMessage, number: int) -> str: + try: + name = part.get_filename() + except Exception: + name = None + if not name: + ext = ".eml" if part.get_content_type() == "message/rfc822" else "" + name = f"attachment-{number}{ext}" + return name + + +def parse(data: bytes) -> ParsedMail: + msg = message_from_bytes(data, policy=policy.default) + if not isinstance(msg, EmailMessage) or not (msg.keys() or msg.get_payload()): + raise ValidationError("Not a readable e-mail: no headers or body found.") + out = ParsedMail() + out.headers = { + "from": _header(msg, "from"), + "to": _addresses(msg, "to"), + "cc": _addresses(msg, "cc"), + "date": _date(msg), + "subject": _header(msg, "subject"), + "message_id": _header(msg, "message-id"), + } + body_part: Optional[EmailMessage] = msg.get_body(preferencelist=("plain", "html")) + if body_part is None: + out.warnings.append("The mail has no text/plain or text/html body (RTF-only or empty); the text is empty.") + elif body_part.get_content_type() == "text/html": + out.body = web_capture.derive_text(_text_of(body_part), "").strip() + else: + out.body = _text_of(body_part).replace("\r\n", "\n").replace("\r", "\n").strip() + for number, part in enumerate(msg.walk(), 1): + if part.is_multipart() or part is body_part or part is msg: + continue + disposition = (part.get_content_disposition() or "").lower() + if ( + disposition != "attachment" + and part.get_content_maintype() == "image" + and part["content-id"] is not None + ): + out.inline_images += 1 + continue + if part.get_content_type() == "message/rfc822": + inner = part.get_payload() + payload = inner[0].as_bytes(policy=policy.SMTP) if inner else b"" + else: + payload = part.get_payload(decode=True) or b"" + out.attachments.append(Attachment(_filename(part, number), payload, part.get_content_type())) + if out.inline_images: + out.warnings.append(f"{out.inline_images} inline image(s) are not in the text.") + return out + + +def mail_fields(parsed: ParsedMail) -> tuple[tuple[str, "str | list[str]"], ...]: + h = parsed.headers + return ( + ("from", h["from"]), + ("to", h["to"]), + ("cc", h["cc"]), + ("date", h["date"]), + ("subject", h["subject"]), + ("message_id", h["message_id"]), + ("attachments", [a.name for a in parsed.attachments]), + ) + + +def mail_checks(parsed: ParsedMail) -> tuple[str, ...]: + checks = [] + if _QUOTED.search(parsed.body): + checks.append("Quoted earlier messages in the body are not the sender's words - attribute only the new text.") + checks.append("A mail often opens or closes a commitment - see wiki-ingest step 4.") + checks.append("Remove secrets before raw accept (raw/CONTRACT.md, 'What does not belong here').") + return tuple(checks) + + +class EmlFilter(Filter): + name = "eml" + extensions = (".eml",) + + def derive(self, data: bytes, name: str) -> Derivation: + parsed = parse(data) + if parsed.attachments: + names = ", ".join(a.name for a in parsed.attachments) + raise ValidationError( + f"The mail carries {len(parsed.attachments)} attachment(s) ({names}); deriving a mail " + "with attachments is not supported yet. Save the attachments as files beside the " + "mail and derive each file on its own." + ) + return Derivation( + text=parsed.body, + title=parsed.headers["subject"] if isinstance(parsed.headers["subject"], str) else "", + fields=mail_fields(parsed), + warnings=tuple(parsed.warnings), + checks=mail_checks(parsed), + ) + + +register(EmlFilter()) diff --git a/tools/chemenu/filters/ooxml.py b/tools/chemenu/filters/ooxml.py new file mode 100644 index 0000000..94eeacf --- /dev/null +++ b/tools/chemenu/filters/ooxml.py @@ -0,0 +1,673 @@ +"""Office Open XML: DOCX, PPTX and XLSX. A package is a ZIP of XML parts, so the +standard library reads all three; no third-party package is involved. + +Tag and attribute names are matched by their local name, which keeps the +"transitional" and "strict" namespace flavours of the format one code path. +""" +from __future__ import annotations + +import datetime +import io +import posixpath +import re +import xml.etree.ElementTree as ET +import zipfile +from typing import Iterator, Optional + +from chemenu.errors import ValidationError +from chemenu.filters import Derivation, Filter, register + +PART_LIMIT = 100 * 1024 * 1024 +CELL_LIMIT = 2_000_000 + + +def _local(tag) -> str: + return tag.rsplit("}", 1)[-1] if isinstance(tag, str) else "" + + +def _attr(el: ET.Element, name: str) -> Optional[str]: + for key, value in el.attrib.items(): + if key.rsplit("}", 1)[-1] == name: + return value + return None + + +def _rid(el: ET.Element) -> Optional[str]: + """The namespaced `r:id`, never the plain `id` an element may carry beside it.""" + for key, value in el.attrib.items(): + if key.endswith("}id"): + return value + return None + + +def _children(el: ET.Element, name: str) -> list[ET.Element]: + return [c for c in el if _local(c.tag) == name] + + +def _child(el: ET.Element, name: str) -> Optional[ET.Element]: + for c in el: + if _local(c.tag) == name: + return c + return None + + +def _cell(text: str) -> str: + return re.sub(r"\s+", " ", text).strip().replace("|", "\\|") + + +def _table(rows: list[list[str]]) -> str: + width = max((len(r) for r in rows), default=0) + if not width: + return "" + padded = [[_cell(c) for c in r] + [""] * (width - len(r)) for r in rows] + lines = ["| " + " | ".join(padded[0]) + " |", "|" + " --- |" * width] + lines += ["| " + " | ".join(r) + " |" for r in padded[1:]] + return "\n".join(lines) + + +class _Package: + def __init__(self, data: bytes, kind: str) -> None: + self.kind = kind + try: + self.zf = zipfile.ZipFile(io.BytesIO(data)) + except zipfile.BadZipFile: + raise ValidationError( + f"Not a readable {kind} file: it is not a ZIP package. A password-protected " + "Office file is not one - remove the password in the application and save a copy." + ) from None + self.names = set(self.zf.namelist()) + + def has(self, name: str) -> bool: + return name in self.names + + def read(self, name: str) -> bytes: + try: + info = self.zf.getinfo(name) + except KeyError: + raise ValidationError(f"The {self.kind} file has no part {name}.") from None + if info.flag_bits & 0x1: + raise ValidationError(f"The {self.kind} file is encrypted - part {name} cannot be read.") + if info.file_size > PART_LIMIT: + raise ValidationError(f"Part {name} of the {self.kind} file is over 100 MiB unpacked.") + try: + return self.zf.read(name) + except (zipfile.BadZipFile, RuntimeError, OSError, EOFError, NotImplementedError) as exc: + raise ValidationError(f"Part {name} of the {self.kind} file is unreadable: {exc}") from None + + def xml(self, name: str) -> ET.Element: + try: + return ET.fromstring(self.read(name)) + except ET.ParseError as exc: + raise ValidationError(f"Part {name} of the {self.kind} file is not valid XML: {exc}") from None + + def rels(self, part: str) -> dict[str, tuple[str, str]]: + """`{id: (type, absolute part name)}` of the relationships a part declares.""" + directory, base = posixpath.split(part) + rels_name = posixpath.join(directory, "_rels", base + ".rels") + if not self.has(rels_name): + return {} + out = {} + for rel in self.xml(rels_name): + if _local(rel.tag) != "Relationship" or rel.get("TargetMode") == "External": + continue + target = rel.get("Target") or "" + absolute = target.lstrip("/") if target.startswith("/") else posixpath.normpath( + posixpath.join(directory, target) + ) + out[rel.get("Id") or ""] = (rel.get("Type") or "", absolute) + return out + + def main_part(self, default: str) -> str: + for _id, (kind, target) in self.rels("").items(): + if kind.endswith("/officeDocument") and self.has(target): + return target + if self.has(default): + return default + raise ValidationError(f"The {self.kind} file has no main document part ({default}).") + + def title(self) -> str: + if not self.has("docProps/core.xml"): + return "" + try: + core = self.xml("docProps/core.xml") + except ValidationError: + return "" + node = _child(core, "title") + return (node.text or "").strip() if node is not None else "" + + +# --- DOCX --------------------------------------------------------------------- + +_TRACKED = {"ins", "del", "moveFrom", "moveTo"} + + +class _Docx: + def __init__(self, pkg: _Package, main: str) -> None: + self.pkg = pkg + self.root = pkg.xml(main) + directory = posixpath.dirname(main) + self.styles = self._styles(posixpath.join(directory, "styles.xml")) + self.numbering = self._numbering(posixpath.join(directory, "numbering.xml")) + self.counters: dict[tuple[str, int], int] = {} + self.tracked = sum(1 for e in self.root.iter() if _local(e.tag) in _TRACKED) + + def _styles(self, name: str) -> dict[str, str]: + if not self.pkg.has(name): + return {} + out = {} + for style in self.pkg.xml(name).iter(): + if _local(style.tag) == "style": + label = _child(style, "name") + out[_attr(style, "styleId") or ""] = (_attr(label, "val") or "").lower() if label is not None else "" + return out + + def _numbering(self, name: str) -> dict[str, dict[int, str]]: + """`{numId: {ilvl: numFmt}}`.""" + if not self.pkg.has(name): + return {} + root = self.pkg.xml(name) + abstract: dict[str, dict[int, str]] = {} + for node in _children(root, "abstractNum"): + levels = {} + for lvl in _children(node, "lvl"): + fmt = _child(lvl, "numFmt") + levels[int(_attr(lvl, "ilvl") or 0)] = (_attr(fmt, "val") or "") if fmt is not None else "" + abstract[_attr(node, "abstractNumId") or ""] = levels + out = {} + for num in _children(root, "num"): + ref = _child(num, "abstractNumId") + if ref is not None: + out[_attr(num, "numId") or ""] = abstract.get(_attr(ref, "val") or "", {}) + return out + + # inline text --------------------------------------------------------------- + + def _run(self, run: ET.Element, out: list[str]) -> None: + for c in run: + tag = _local(c.tag) + if tag == "t": + out.append(c.text or "") + elif tag == "tab": + out.append("\t") + elif tag == "br": + if _attr(c, "type") != "page": + out.append("\n") + elif tag == "cr": + out.append("\n") + elif tag == "noBreakHyphen": + out.append("-") + + def _inline(self, el: ET.Element, out: list[str]) -> None: + for c in el: + tag = _local(c.tag) + if tag == "r": + self._run(c, out) + elif tag in ("ins", "moveTo", "hyperlink", "smartTag", "fldSimple", "sdt", "sdtContent", "customXml"): + self._inline(c, out) + + def paragraph_text(self, p: ET.Element, hard_breaks: bool = False) -> str: + parts: list[str] = [] + self._inline(p, parts) + text = "".join(parts).replace("\t", " ") + text = re.sub(r"[ ]*\n[ ]*", " \n" if hard_breaks else " ", text) + return text.strip() + + # blocks -------------------------------------------------------------------- + + def _heading_level(self, p: ET.Element) -> Optional[int]: + ppr = _child(p, "pPr") + style = _child(ppr, "pStyle") if ppr is not None else None + if style is None: + return None + style_id = _attr(style, "val") or "" + label = self.styles.get(style_id) or style_id.lower() + match = re.fullmatch(r"heading\s*(\d)", label) + return min(int(match.group(1)), 6) if match and int(match.group(1)) else None + + def _list_marker(self, p: ET.Element) -> Optional[tuple[int, str]]: + ppr = _child(p, "pPr") + num = _child(ppr, "numPr") if ppr is not None else None + if num is None: + return None + num_id_node, lvl_node = _child(num, "numId"), _child(num, "ilvl") + num_id = (_attr(num_id_node, "val") or "") if num_id_node is not None else "" + if num_id in ("", "0"): + return None + level = int(_attr(lvl_node, "val") or 0) if lvl_node is not None else 0 + fmt = self.numbering.get(num_id, {}).get(level, "bullet") + if fmt in ("bullet", "none"): + return level, "-" + for key in [k for k in self.counters if k[0] == num_id and k[1] > level]: + del self.counters[key] + self.counters[(num_id, level)] = self.counters.get((num_id, level), 0) + 1 + return level, f"{self.counters[(num_id, level)]}." + + def _cell_text(self, tc: ET.Element) -> str: + parts = [] + for c in tc: + tag = _local(c.tag) + if tag == "p": + parts.append(self.paragraph_text(c)) + elif tag == "tbl": + parts.append(" ".join(self._cell_text(inner) for row in _children(c, "tr") for inner in _children(row, "tc"))) + return " ".join(p for p in parts if p) + + def _rows(self, tbl: ET.Element) -> list[list[str]]: + rows = [] + for tr in _children(tbl, "tr"): + cells: list[str] = [] + for tc in _children(tr, "tc"): + props = _child(tc, "tcPr") + span = _child(props, "gridSpan") if props is not None else None + cells.append(self._cell_text(tc)) + cells.extend([""] * (int(_attr(span, "val") or 1) - 1 if span is not None else 0)) + rows.append(cells) + return rows + + def blocks(self, container: ET.Element) -> Iterator[tuple[str, str]]: + for child in container: + tag = _local(child.tag) + if tag == "p": + level = self._heading_level(child) + if level is not None: + text = self.paragraph_text(child) + if text: + yield "block", "#" * level + " " + text + continue + marker = self._list_marker(child) + if marker is not None: + text = self.paragraph_text(child) + if text: + yield "item", " " * marker[0] + marker[1] + " " + text + continue + text = self.paragraph_text(child, hard_breaks=True) + if text: + yield "block", text + elif tag == "tbl": + table = _table(self._rows(child)) + if table: + yield "block", table + elif tag in ("sdt", "sdtContent"): + yield from self.blocks(child) + + def comments(self) -> list[tuple[str, str, str]]: + name = posixpath.join(posixpath.dirname(self.pkg.main_part("word/document.xml")), "comments.xml") + if not self.pkg.has(name): + return [] + out = [] + for comment in _children(self.pkg.xml(name), "comment"): + text = " ".join(t for t in (self.paragraph_text(p) for p in comment.iter() if _local(p.tag) == "p") if t) + out.append((_attr(comment, "author") or "", _attr(comment, "date") or "", text)) + return out + + +def _join_blocks(blocks: list[tuple[str, str]]) -> str: + out: list[str] = [] + previous = "" + for kind, text in blocks: + if out: + out.append("\n" if kind == "item" and previous == "item" else "\n\n") + out.append(text) + previous = kind + return "".join(out) + + +class DocxFilter(Filter): + name = "docx" + extensions = (".docx",) + + def derive(self, data: bytes, name: str) -> Derivation: + pkg = _Package(data, "DOCX") + main = pkg.main_part("word/document.xml") + doc = _Docx(pkg, main) + body = _child(doc.root, "body") + if body is None: + raise ValidationError("The DOCX file has no document body.") + blocks = list(doc.blocks(body)) + comments = doc.comments() + if comments: + blocks.append(("block", "## Comments")) + for author, date, text in comments: + label = ", ".join(x for x in (author, date) if x) + blocks.append(("item", f"- {label}: {text}" if label else f"- {text}")) + checks = [] + if doc.tracked: + checks.append( + "Tracked changes are taken as accepted (insertions in, deletions out) - the " + "original still holds the history." + ) + if comments: + checks.append("Comments sit under '## Comments' at the end - discussion, not the document's claims.") + return Derivation( + text=_join_blocks(blocks), + title=pkg.title(), + fields=(("tracked_changes", str(doc.tracked)), ("comments", str(len(comments)))), + checks=tuple(checks), + ) + + +# --- PPTX --------------------------------------------------------------------- + +_SKIPPED_PLACEHOLDERS = {"sldNum", "dt", "ftr", "hdr"} +_TITLES = {"title", "ctrTitle"} + + +def _a_paragraph(p: ET.Element) -> str: + parts = [] + for c in p: + tag = _local(c.tag) + if tag in ("r", "fld"): + parts.extend((t.text or "") for t in c if _local(t.tag) == "t") + elif tag == "br": + parts.append(" ") + return re.sub(r"\s+", " ", "".join(parts)).strip() + + +def _placeholder(shape: ET.Element) -> Optional[str]: + for node in shape.iter(): + if _local(node.tag) == "ph": + return _attr(node, "type") or "body" + return None + + +class _Slide: + def __init__(self) -> None: + self.blocks: list[str] = [] + self.opaque = 0 + + def shapes(self, tree: ET.Element) -> None: + for child in tree: + tag = _local(child.tag) + if tag == "sp": + self.shape(child) + elif tag in ("grpSp", "AlternateContent", "Choice"): + self.shapes(child) + elif tag == "graphicFrame": + tbl = next((e for e in child.iter() if _local(e.tag) == "tbl"), None) + if tbl is None: + self.opaque += 1 + else: + rows = [ + [" ".join(_a_paragraph(p) for p in tc.iter() if _local(p.tag) == "p") for tc in _children(tr, "tc")] + for tr in _children(tbl, "tr") + ] + table = _table(rows) + if table: + self.blocks.append(table) + elif tag == "pic": + self.opaque += 1 + + def shape(self, sp: ET.Element) -> None: + kind = _placeholder(sp) + if kind in _SKIPPED_PLACEHOLDERS: + return + body = _child(sp, "txBody") + if body is None: + return + paragraphs = [(int(_attr(_child(p, "pPr"), "lvl") or 0) if _child(p, "pPr") is not None else 0, _a_paragraph(p)) + for p in _children(body, "p")] + paragraphs = [(lvl, text) for lvl, text in paragraphs if text] + if not paragraphs: + return + if kind in _TITLES: + self.blocks.append("**" + " ".join(text for _lvl, text in paragraphs) + "**") + elif len(paragraphs) == 1: + self.blocks.append(paragraphs[0][1]) + else: + self.blocks.append("\n".join(" " * lvl + "- " + text for lvl, text in paragraphs)) + + +class PptxFilter(Filter): + name = "pptx" + extensions = (".pptx",) + + def derive(self, data: bytes, name: str) -> Derivation: + pkg = _Package(data, "PPTX") + main = pkg.main_part("ppt/presentation.xml") + rels = pkg.rels(main) + slide_parts = [] + for node in pkg.xml(main).iter(): + if _local(node.tag) == "sldId": + target = rels.get(_rid(node) or "") + if target and pkg.has(target[1]): + slide_parts.append(target[1]) + if not slide_parts: + raise ValidationError("The PPTX file lists no readable slides.") + sections: list[str] = [] + opaque = 0 + for number, part in enumerate(slide_parts, 1): + slide = _Slide() + tree = next((e for e in pkg.xml(part).iter() if _local(e.tag) == "spTree"), None) + if tree is not None: + slide.shapes(tree) + opaque += slide.opaque + block = [f"## Slide {number}"] + slide.blocks + notes = self._notes(pkg, part) + if notes: + block += ["### Notes", notes] + sections.append("\n\n".join(block)) + checks = [] + if opaque: + checks.append( + "Pictures, charts and diagrams are not in the text - open the original where a " + "slide's meaning rests on them." + ) + return Derivation( + text="\n\n".join(sections), + title=pkg.title(), + fields=(("slides", str(len(slide_parts))),), + checks=tuple(checks), + ) + + def _notes(self, pkg: _Package, slide_part: str) -> str: + for kind, target in pkg.rels(slide_part).values(): + if kind.endswith("/notesSlide") and pkg.has(target): + tree = next((e for e in pkg.xml(target).iter() if _local(e.tag) == "spTree"), None) + if tree is None: + return "" + texts = [] + for sp in (e for e in tree.iter() if _local(e.tag) == "sp"): + if _placeholder(sp) == "body": + body = _child(sp, "txBody") + if body is not None: + texts += [t for t in (_a_paragraph(p) for p in _children(body, "p")) if t] + return "\n\n".join(texts) + return "" + + +# --- XLSX --------------------------------------------------------------------- + +_BUILTIN_DATE = {**{i: (True, False) for i in (*range(14, 18), *range(27, 37), *range(50, 59))}, + **{i: (False, True) for i in (18, 19, 20, 21, 45, 46, 47)}, 22: (True, True)} + + +def _col_index(letters: str) -> int: + n = 0 + for ch in letters: + n = n * 26 + ord(ch) - 64 + return n + + +def _format_kind(code: str) -> Optional[tuple[bool, bool]]: + code = re.sub(r'"[^"]*"|\[[^\]]*\]|\\.|_.|\*.', "", code).lower() + has_time = "h" in code or "s" in code or "am/pm" in code + has_date = "y" in code or "d" in code or ("m" in code and not has_time) + return (has_date, has_time) if has_date or has_time else None + + +class _Workbook: + def __init__(self, pkg: _Package, main: str) -> None: + self.pkg = pkg + self.directory = posixpath.dirname(main) + self.root = pkg.xml(main) + self.rels = pkg.rels(main) + self.date1904 = any(_local(e.tag) == "workbookPr" and _attr(e, "date1904") in ("1", "true") for e in self.root.iter()) + self.shared = self._shared_strings() + self.date_styles = self._date_styles() + self.formulas_without_value = 0 + + def _shared_strings(self) -> list[str]: + for kind, target in self.rels.values(): + if kind.endswith("/sharedStrings") and self.pkg.has(target): + return [self._si_text(si) for si in _children(self.pkg.xml(target), "si")] + return [] + + def _si_text(self, si: ET.Element) -> str: + parts = [] + for c in si: + tag = _local(c.tag) + if tag == "t": + parts.append(c.text or "") + elif tag == "r": + parts += [(t.text or "") for t in c if _local(t.tag) == "t"] + return "".join(parts) + + def _date_styles(self) -> dict[int, tuple[bool, bool]]: + """`{cellXfs index: (has_date, has_time)}` for the styles that format a date.""" + for kind, target in self.rels.values(): + if kind.endswith("/styles") and self.pkg.has(target): + root = self.pkg.xml(target) + custom = {} + for fmts in _children(root, "numFmts"): + for fmt in _children(fmts, "numFmt"): + found = _format_kind(_attr(fmt, "formatCode") or "") + if found: + custom[int(_attr(fmt, "numFmtId") or -1)] = found + xfs = _child(root, "cellXfs") + out = {} + for index, xf in enumerate(_children(xfs, "xf") if xfs is not None else []): + fmt_id = int(_attr(xf, "numFmtId") or 0) + found = custom.get(fmt_id) or _BUILTIN_DATE.get(fmt_id) + if found: + out[index] = found + return out + return {} + + def sheets(self) -> list[tuple[str, str, str]]: + """`(name, state, part)` in sheet order.""" + out = [] + for node in self.root.iter(): + if _local(node.tag) == "sheet": + target = self.rels.get(_rid(node) or "") + out.append((_attr(node, "name") or "", _attr(node, "state") or "visible", target[1] if target else "")) + return out + + def _date(self, raw: str, kind: tuple[bool, bool]) -> str: + try: + serial = float(raw) + except ValueError: + return raw + seconds = round(serial * 86400) + has_date, has_time = kind + if not has_date: + return f"{seconds // 3600:02d}:{seconds % 3600 // 60:02d}:{seconds % 60:02d}" + if self.date1904: + base = datetime.datetime(1904, 1, 1) + elif serial < 60: + base = datetime.datetime(1899, 12, 31) + elif serial < 61: + return raw + else: + base = datetime.datetime(1899, 12, 30) + try: + moment = base + datetime.timedelta(seconds=seconds) + except OverflowError: + return raw + return moment.strftime("%Y-%m-%d %H:%M:%S" if has_time else "%Y-%m-%d") + + def _value(self, c: ET.Element) -> str: + kind = c.get("t") or "n" + v = _child(c, "v") + raw = (v.text or "") if v is not None else "" + has_formula = _child(c, "f") is not None + if kind == "inlineStr": + node = _child(c, "is") + text = self._si_text(node) if node is not None else "" + elif kind == "s": + text = self.shared[int(raw)] if raw.isdigit() and int(raw) < len(self.shared) else "" + elif kind == "b": + text = "" if raw == "" else ("TRUE" if raw == "1" else "FALSE") + elif kind in ("str", "e", "d"): + text = raw + else: + style = int(c.get("s") or 0) + text = self._date(raw, self.date_styles[style]) if raw and style in self.date_styles else raw + if has_formula and raw == "" and kind != "inlineStr": + self.formulas_without_value += 1 + return text + + def table(self, part: str) -> str: + cells: dict[tuple[int, int], str] = {} + row_no, last_col = 0, 0 + for _event, el in ET.iterparse(io.BytesIO(self.pkg.read(part)), events=("end",)): + tag = _local(el.tag) + if tag == "c": + ref = re.fullmatch(r"([A-Za-z]+)(\d+)", el.get("r") or "") + if ref: + col, row = _col_index(ref.group(1).upper()), int(ref.group(2)) + else: + col, row = last_col + 1, row_no + 1 + last_col = col + text = self._value(el) + if text.strip(): + cells[(row, col)] = text + el.clear() + elif tag == "row": + row_no = int(el.get("r") or row_no + 1) + last_col = 0 + el.clear() + if not cells: + return "*(empty sheet)*" + rows = [r for r, _c in cells] + cols = [c for _r, c in cells] + min_r, max_r, min_c, max_c = min(rows), max(rows), min(cols), max(cols) + if (max_r - min_r + 1) * (max_c - min_c + 1) > CELL_LIMIT: + raise ValidationError( + f"A sheet spans {max_r - min_r + 1} rows by {max_c - min_c + 1} columns - too large " + "or too sparse to derive." + ) + grid = [[cells.get((r, c), "") for c in range(min_c, max_c + 1)] for r in range(min_r, max_r + 1)] + return _table(grid) + + +class XlsxFilter(Filter): + name = "xlsx" + extensions = (".xlsx", ".xlsm") + + def derive(self, data: bytes, name: str) -> Derivation: + pkg = _Package(data, "XLSX") + book = _Workbook(pkg, pkg.main_part("xl/workbook.xml")) + sheets = book.sheets() + if not sheets: + raise ValidationError("The XLSX file lists no sheets.") + sections, hidden, warnings = [], [], [] + for sheet_name, state, part in sheets: + if state != "visible": + hidden.append(sheet_name) + if not part or not pkg.has(part): + warnings.append(f"Sheet '{sheet_name}' has no readable worksheet part and was left out.") + continue + if _child(pkg.xml(part), "sheetData") is None and _local(pkg.xml(part).tag) == "chartsheet": + sections.append(f"## {sheet_name}\n\n*(chart sheet, no cell data)*") + continue + sections.append(f"## {sheet_name}\n\n{book.table(part)}") + if book.formulas_without_value: + warnings.append( + f"{book.formulas_without_value} formula cell(s) have no stored value (the file was " + "never calculated) and are empty." + ) + checks = ["Values are as last saved; formulas were not recalculated."] + if hidden: + checks.append("Hidden sheets are included and named in hidden_sheets.") + return Derivation( + text="\n\n".join(sections), + title=pkg.title(), + fields=(("sheets", str(len(sheets))), ("hidden_sheets", hidden)), + warnings=tuple(warnings), + checks=tuple(checks), + ) + + +register(DocxFilter()) +register(PptxFilter()) +register(XlsxFilter()) diff --git a/tools/chemenu/tests/test_raw_derive.py b/tools/chemenu/tests/test_raw_derive.py new file mode 100644 index 0000000..49e506a --- /dev/null +++ b/tools/chemenu/tests/test_raw_derive.py @@ -0,0 +1,676 @@ +"""`wikitool raw derive` - reading text from documents that are not text. + +The documents are built here from `zipfile` and `email`, so every fixture is +readable in the test that uses it and no binary sits in the repository. +""" +from __future__ import annotations + +import io +import json +import zipfile +from email.message import EmailMessage +from pathlib import Path + +import pytest +import typer +import yaml + +from chemenu import filters, web_capture +from chemenu.commands.raw_cmd import ( + raw_accept_command, + raw_derive_command, + raw_pending_command, +) + +W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +A = "http://schemas.openxmlformats.org/drawingml/2006/main" +P = "http://schemas.openxmlformats.org/presentationml/2006/main" +S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" +R = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" +REL = "http://schemas.openxmlformats.org/package/2006/relationships" +DOC_REL = R + "/officeDocument" + + +def _zip(parts: dict[str, str]) -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as zf: + for name, content in parts.items(): + zf.writestr(name, content) + return buffer.getvalue() + + +def _rels(target: str, kind: str = DOC_REL, extra: str = "") -> str: + return ( + f'{extra}' + ) + + +def make_docx(body: str, *, comments: str = "", title: str = "") -> bytes: + parts = { + "_rels/.rels": _rels("word/document.xml"), + "word/document.xml": f'{body}', + "word/styles.xml": ( + f'' + "" + ), + "word/numbering.xml": ( + f'' + '' + '' + '' + '' + ), + } + if comments: + parts["word/comments.xml"] = f'{comments}' + if title: + parts["docProps/core.xml"] = ( + '' + f"{title}" + ) + return _zip(parts) + + +def _p(text: str, *, style: str = "", num: tuple[int, int] | None = None) -> str: + props = "" + if style: + props += f'' + if num: + props += f'' + return f"{props}{text}" + + +def make_pptx(slides: list[tuple[str, list[str], str]]) -> bytes: + """`(title, bullets, notes)` per slide.""" + parts = {"_rels/.rels": _rels("ppt/presentation.xml")} + ids, rels = "", "" + for number, (title, bullets, notes) in enumerate(slides, 1): + ids += f'' + rels += ( + f'' + ) + body = "".join(f"{b}" for b in bullets) + parts[f"ppt/slides/slide{number}.xml"] = ( + f'' + f'' + f"{title}" + f'{body}' + f'' + f"FOOTER-NUMBER" + "" + ) + if notes: + parts[f"ppt/slides/_rels/slide{number}.xml.rels"] = ( + f'' + ) + parts[f"ppt/notesSlides/notes{number}.xml"] = ( + f'' + '' + f"{notes}" + ) + parts["ppt/presentation.xml"] = ( + f'{ids}' + ) + parts["ppt/_rels/presentation.xml.rels"] = f'{rels}' + return _zip(parts) + + +def make_xlsx() -> bytes: + sheet_rel = lambda n: f'' # noqa: E731 + return _zip({ + "_rels/.rels": _rels("xl/workbook.xml"), + "xl/workbook.xml": ( + f'' + '' + '' + ), + "xl/_rels/workbook.xml.rels": ( + f'{sheet_rel(1)}{sheet_rel(2)}' + f'' + f'' + ), + "xl/sharedStrings.xml": ( + f'PostenMiete' + "Wert" + ), + "xl/styles.xml": ( + f'' + ), + "xl/worksheets/sheet1.xml": ( + f'' + '02' + '11200+01200' + 'Stichtag45292' + 'Fertig1' + "" + ), + "xl/worksheets/sheet2.xml": ( + f'' + "VERSTECKT-INHALT" + ), + }) + + +def make_eml(body: str = "Hallo Welt\n", *, html: str = "", attach: bool = False, inline: bool = False, + subject: str = "Angebot") -> bytes: + msg = EmailMessage() + msg["From"] = "Anna Beispiel " + msg["To"] = "bob@example.org, Carla " + msg["Subject"] = subject + msg["Date"] = "Wed, 07 Oct 2026 09:30:00 +0200" + msg["Message-ID"] = "" + if html: + msg.set_content(html, subtype="html") + else: + msg.set_content(body) + if inline: + msg.add_related(b"\x89PNG", maintype="image", subtype="png", cid="") + if attach: + msg.add_attachment(b"data", maintype="application", subtype="octet-stream", filename="a.bin") + return msg.as_bytes() + + +@pytest.fixture +def tree(kb_dir): + root = kb_dir.parent + (root / "raw").mkdir() + (root / "incoming").mkdir(exist_ok=True) + return root + + +def _derive(path, name=None): + return raw_derive_command(path=Path(path), name=name) + + +def _refused(path, name=None) -> None: + with pytest.raises(typer.Exit) as excinfo: + _derive(path, name) + assert excinfo.value.exit_code == 1 + + +def _snapshot(root: Path) -> dict[str, bytes]: + return {str(p.relative_to(root)): p.read_bytes() for p in sorted(root.rglob("*")) if p.is_file()} + + +def _header_and_text(md: Path) -> tuple[dict, str]: + content = md.read_text(encoding="utf-8") + assert content.startswith("---\n") + head, _, text = content[4:].partition("\n---\n") + return yaml.safe_load(head), text + + +def _put(tree: Path, name: str, data: bytes) -> Path: + path = tree / "incoming" / name + path.write_bytes(data) + return path + + +# --- the single-file form --------------------------------------------------- + + +def test_a_single_file_becomes_a_folder_with_original_and_derived_text(tree): + data = make_docx(_p("Hello")) + _put(tree, "Angebot XYZ.docx", data) + + _derive("incoming/Angebot XYZ.docx") + + folder = tree / "incoming/Angebot XYZ" + assert sorted(p.name for p in folder.iterdir()) == ["derived.md", "original.docx"] + assert (folder / "original.docx").read_bytes() == data + assert not (tree / "incoming/Angebot XYZ.docx").exists() + + +def test_the_header_is_valid_yaml_and_records_the_derivation(tree): + _put(tree, "Angebot XYZ.docx", make_docx(_p("Hello"), title="Offer")) + _derive("incoming/Angebot XYZ.docx") + header, text = _header_and_text(tree / "incoming/Angebot XYZ/derived.md") + + assert header["derived_by"] == "wikitool raw derive" + assert header["filter"] == "docx" + assert header["engine"].startswith("wikitool ") + assert header["derived_from"] == "original.docx" + assert header["original_name"] == "Angebot XYZ.docx" + assert header["title"] == "Offer" + assert header["derived"] is not None + assert text.strip() == "Hello" + + +def test_two_derivations_of_the_same_bytes_differ_only_in_stamp_and_file_name(tree): + data = make_docx(_p("Hello") + _p("World")) + _put(tree, "a.docx", data) + _put(tree, "b.docx", data) + _derive("incoming/a.docx", "same") + _derive("incoming/b.docx", "same-too") + + def stripped(name): + lines = (tree / "incoming" / name / "derived.md").read_text(encoding="utf-8").splitlines() + return [line for line in lines if not line.startswith(("derived:", "original_name:"))] + + assert stripped("same") == stripped("same-too") + + +def test_raw_accept_takes_the_derived_folder_as_one_source(tree): + data = make_docx(_p("Hello")) + _put(tree, "report.docx", data) + _derive("incoming/report.docx") + + raw_accept_command( + files=[Path("incoming/report")], fidelity="verbatim", authority="reporting", + page=None, replaces=None, dry_run=False, + ) + + [original] = list((tree / "raw").rglob("original.docx")) + assert original.read_bytes() == data + assert (original.parent / "derived.md").is_file() + assert original.parent.name == "report" + assert not (tree / "incoming/report").exists() + + +def test_name_picks_the_folder_stem(tree): + _put(tree, "x.docx", make_docx(_p("Hello"))) + _derive("incoming/x.docx", "kurz") + assert (tree / "incoming/kurz/original.docx").is_file() + + +def test_a_120_character_stem_fits_and_an_over_budget_one_is_refused(tree, capsys): + long_stem = "s" * 120 + _put(tree, f"{long_stem}.docx", make_docx(_p("Hello"))) + _derive(f"incoming/{long_stem}.docx") + assert (tree / "incoming" / long_stem / "derived.md").is_file() + capsys.readouterr() + + too_long = "t" * 170 + _put(tree, f"{too_long}.docx", make_docx(_p("Hello"))) + before = _snapshot(tree / "incoming") + _refused(f"incoming/{too_long}.docx") + out = capsys.readouterr().out + assert "--name" in out and "raw/" in out + assert _snapshot(tree / "incoming") == before + _derive(f"incoming/{too_long}.docx", "short") + assert (tree / "incoming/short/original.docx").is_file() + + +def test_a_stem_taken_under_raw_is_a_warning_not_a_refusal(tree, capsys): + (tree / "raw/2026/01").mkdir(parents=True) + (tree / "raw/2026/01/report.txt").write_text("x", encoding="utf-8") + _put(tree, "report.docx", make_docx(_p("Hello"))) + _derive("incoming/report.docx") + out = capsys.readouterr().out + assert "WARN" in out and "already claims" in out + assert (tree / "incoming/report/derived.md").is_file() + + +def test_an_existing_target_folder_is_refused_and_nothing_changes(tree): + _put(tree, "report.docx", make_docx(_p("Hello"))) + (tree / "incoming/report").mkdir() + before = _snapshot(tree / "incoming") + _refused("incoming/report.docx") + assert _snapshot(tree / "incoming") == before + + +def test_a_file_outside_incoming_or_in_a_subfolder_is_refused(tree): + outside = tree / "elsewhere.docx" + outside.write_bytes(make_docx(_p("Hello"))) + _refused(outside) + (tree / "incoming/sub").mkdir() + (tree / "incoming/sub/in.docx").write_bytes(make_docx(_p("Hello"))) + _refused("incoming/sub/in.docx") + assert (tree / "incoming/sub/in.docx").is_file() + + +def test_a_file_without_a_filter_is_refused_and_names_what_is_read(tree, capsys): + _put(tree, "notes.md", b"# notes\n") + _refused("incoming/notes.md") + out = capsys.readouterr().out + assert ".docx" in out and ".xlsx" in out + assert (tree / "incoming/notes.md").is_file() + + +def test_a_failure_while_moving_the_original_leaves_incoming_as_it_was(tree, monkeypatch): + _put(tree, "report.docx", make_docx(_p("Hello"))) + before = _snapshot(tree / "incoming") + + def broken(self, target): + raise OSError("disk says no") + + monkeypatch.setattr(Path, "rename", broken) + _refused("incoming/report.docx") + monkeypatch.undo() + assert _snapshot(tree / "incoming") == before + assert not (tree / "incoming/report").exists() + + +def test_a_damaged_document_is_refused_untouched(tree): + _put(tree, "broken.docx", b"this is not a zip") + before = _snapshot(tree / "incoming") + _refused("incoming/broken.docx") + assert _snapshot(tree / "incoming") == before + + +def test_a_lone_markdown_file_stays_folderless(tree): + _put(tree, "notes.md", b"# notes\n") + raw_accept_command( + files=[tree / "incoming/notes.md"], fidelity="verbatim", authority="opinion", + page=None, replaces=None, dry_run=False, + ) + [moved] = [p for p in (tree / "raw").rglob("*") if p.is_file()] + assert moved.name == "notes.md" and moved.parent.name.isdigit() + + +# --- the folder form -------------------------------------------------------- + + +def test_a_folder_gets_a_md_beside_each_file_with_a_filter(tree, capsys): + folder = tree / "incoming/mappe" + folder.mkdir() + (folder / "plan.docx").write_bytes(make_docx(_p("Plan"))) + (folder / "zahlen.xlsx").write_bytes(make_xlsx()) + (folder / "foto.jpg").write_bytes(b"\xff\xd8") + _derive("incoming/mappe") + out = capsys.readouterr().out + + assert sorted(p.name for p in folder.iterdir()) == ["foto.jpg", "plan.docx", "plan.md", "zahlen.md", "zahlen.xlsx"] + header, _ = _header_and_text(folder / "plan.md") + assert header["derived_from"] == "plan.docx" + assert "original_name" not in header + assert "foto.jpg" in out + + +def test_a_folder_with_an_existing_md_is_refused_untouched(tree): + folder = tree / "incoming/mappe" + folder.mkdir() + (folder / "plan.docx").write_bytes(make_docx(_p("Plan"))) + (folder / "plan.md").write_text("mine\n", encoding="utf-8") + before = _snapshot(tree / "incoming") + _refused("incoming/mappe") + assert _snapshot(tree / "incoming") == before + + +def test_a_folder_where_two_files_derive_to_one_md_is_refused(tree): + folder = tree / "incoming/mappe" + folder.mkdir() + (folder / "plan.docx").write_bytes(make_docx(_p("Plan"))) + (folder / "plan.xlsx").write_bytes(make_xlsx()) + before = _snapshot(tree / "incoming") + _refused("incoming/mappe") + assert _snapshot(tree / "incoming") == before + + +def test_a_folder_without_any_filter_file_is_refused(tree): + folder = tree / "incoming/mappe" + folder.mkdir() + (folder / "foto.jpg").write_bytes(b"\xff\xd8") + _refused("incoming/mappe") + + +def test_a_folder_with_a_hidden_entry_is_refused(tree): + folder = tree / "incoming/mappe" + folder.mkdir() + (folder / "plan.docx").write_bytes(make_docx(_p("Plan"))) + (folder / ".DS_Store").write_bytes(b"x") + _refused("incoming/mappe") + + +def test_name_does_not_apply_to_a_folder(tree): + (tree / "incoming/mappe").mkdir() + (tree / "incoming/mappe/plan.docx").write_bytes(make_docx(_p("Plan"))) + _refused("incoming/mappe", "other") + + +# --- DOCX ------------------------------------------------------------------- + + +def test_docx_headings_lists_tables_and_text(tree): + body = ( + _p("Titel", style="Titel") + + _p("Absatz eins.") + + _p("Erster", num=(0, 1)) + _p("Zweiter", num=(0, 1)) + + _p("Punkt", num=(0, 2)) + + f'{_p("A")}{_p("B")}' + f'{_p("1")}{_p("2")}' + ) + _put(tree, "d.docx", make_docx(body)) + _derive("incoming/d.docx") + _, text = _header_and_text(tree / "incoming/d/derived.md") + + assert "# Titel" in text + assert "1. Erster\n2. Zweiter" in text + assert "- Punkt" in text + assert "| A | B |" in text and "| 1 | 2 |" in text + + +def test_docx_tracked_changes_and_comments(tree, capsys): + body = ( + "Alt " + 'GELOESCHT ' + 'NEU' + ) + comments = 'Prüfen' + _put(tree, "d.docx", make_docx(body, comments=comments)) + _derive("incoming/d.docx") + out = capsys.readouterr().out + header, text = _header_and_text(tree / "incoming/d/derived.md") + + assert "Alt NEU" in text and "GELOESCHT" not in text + assert "## Comments" in text and "Anna" in text and "Prüfen" in text + assert header["tracked_changes"] == 2 and header["comments"] == 1 + assert "CHECK" in out and "Tracked changes" in out and "Comments" in out + + +def test_docx_without_tracked_changes_prints_no_such_check(tree, capsys): + _put(tree, "d.docx", make_docx(_p("Plain"))) + _derive("incoming/d.docx") + out = capsys.readouterr().out + assert "Tracked changes" not in out and "Comments" not in out + + +# --- PPTX ------------------------------------------------------------------- + + +def test_pptx_slides_notes_and_footer_placeholders(tree): + _put(tree, "deck.pptx", make_pptx([("Eins", ["a", "b"], "Sprechnotiz"), ("Zwei", ["c"], "")])) + _derive("incoming/deck.pptx") + header, text = _header_and_text(tree / "incoming/deck/derived.md") + + assert header["slides"] == 2 + assert text.index("## Slide 1") < text.index("## Slide 2") + assert "**Eins**" in text and "- a" in text and "- b" in text + assert "### Notes" in text and "Sprechnotiz" in text + assert "FOOTER-NUMBER" not in text + + +# --- XLSX ------------------------------------------------------------------- + + +def test_xlsx_values_hidden_sheet_and_trimmed_edges(tree, capsys): + _put(tree, "zahlen.xlsx", make_xlsx()) + _derive("incoming/zahlen.xlsx") + out = capsys.readouterr().out + header, text = _header_and_text(tree / "incoming/zahlen/derived.md") + + assert header["sheets"] == 2 and header["hidden_sheets"] == ["Geheim"] + assert "## Budget" in text and "## Geheim" in text and "VERSTECKT-INHALT" in text + assert "| Posten | Wert |" in text + assert "| Miete | 1200 |" in text + assert "2024-01-01" in text + assert "TRUE" in text + assert text.splitlines()[text.splitlines().index("## Budget") + 2].startswith("| Posten") + assert "Values are as last saved" in out and "Hidden sheets" in out + + +def test_xlsx_check_is_printed_only_for_the_formats_derived_in_the_run(tree, capsys): + _put(tree, "d.docx", make_docx(_p("Plain"))) + _derive("incoming/d.docx") + assert "last saved" not in capsys.readouterr().out + + +def test_check_lines_are_at_most_three_per_format_derived(tree, capsys): + folder = tree / "incoming/mappe" + folder.mkdir() + (folder / "z.xlsx").write_bytes(make_xlsx()) + (folder / "m.eml").write_bytes(make_eml("> quoted\nnew\n")) + _derive("incoming/mappe") + out = capsys.readouterr().out + per_filter = filters.MAX_CHECKS + assert len([line for line in out.splitlines() if line.startswith("CHECK")]) <= 2 * per_filter + + +# --- EML -------------------------------------------------------------------- + + +def test_eml_headers_and_plain_body(tree): + _put(tree, "mail.eml", make_eml("Hallo Welt\n")) + _derive("incoming/mail.eml") + header, text = _header_and_text(tree / "incoming/mail/derived.md") + + assert header["filter"] == "eml" + assert header["from"] == "Anna Beispiel " + assert header["to"] == ["bob@example.org", "Carla "] + assert header["cc"] == [] + assert header["subject"] == "Angebot" and header["title"] == "Angebot" + assert header["message_id"] == "" + assert str(header["date"]).startswith("2026-10-07") + assert header["attachments"] == [] + assert text.strip() == "Hallo Welt" + + +def test_eml_html_only_is_derived_through_the_web_filter(tree): + _put(tree, "mail.eml", make_eml(html="

Gruß

Text

")) + _derive("incoming/mail.eml") + _, text = _header_and_text(tree / "incoming/mail/derived.md") + assert "# Gruß" in text and "Text" in text + + +def test_eml_check_lines_cover_quotes_only_when_they_occur(tree, capsys): + _put(tree, "plain.eml", make_eml("Fresh text\n")) + _derive("incoming/plain.eml") + out = capsys.readouterr().out + assert "Quoted earlier messages" not in out + assert "commitment" in out and "secrets" in out + + _put(tree, "reply.eml", make_eml("Yes.\n\n> earlier\n")) + _derive("incoming/reply.eml") + out = capsys.readouterr().out + assert "Quoted earlier messages" in out + lines = [line for line in out.splitlines() if line.startswith("CHECK")] + assert len(lines) == filters.MAX_CHECKS + + +def test_eml_with_inline_image_warns_and_is_not_an_attachment(tree, capsys): + _put(tree, "mail.eml", make_eml(html="

Hi

", inline=True)) + _derive("incoming/mail.eml") + assert "inline image" in capsys.readouterr().out + + +def test_eml_with_an_attachment_is_refused_untouched(tree, capsys): + _put(tree, "mail.eml", make_eml(attach=True)) + before = _snapshot(tree / "incoming") + _refused("incoming/mail.eml") + assert "a.bin" in capsys.readouterr().out + assert _snapshot(tree / "incoming") == before + + +# --- the registry ----------------------------------------------------------- + + +class _Needy(filters.Filter): + name = "needy" + extensions = (".needy",) + requires = ("no_such_package_for_wikitool",) + + +def test_a_missing_package_fails_that_format_only(tree, monkeypatch, capsys): + monkeypatch.setattr(filters, "_REGISTRY", list(filters.all_filters()) + [_Needy()]) + _put(tree, "x.needy", b"data") + before = _snapshot(tree / "incoming") + _refused("incoming/x.needy") + out = capsys.readouterr().out + assert "no_such_package_for_wikitool" in out and "preflight" in out + assert _snapshot(tree / "incoming") == before + + _put(tree, "ok.docx", make_docx(_p("Fine"))) + _derive("incoming/ok.docx") + assert (tree / "incoming/ok/derived.md").is_file() + + +def test_registering_a_second_reader_for_an_extension_is_refused(): + class Twin(filters.Filter): + name = "twin" + extensions = (".docx",) + + with pytest.raises(ValueError): + filters.register(Twin()) + + +def test_a_filter_cannot_ask_for_more_than_three_check_lines(): + with pytest.raises(ValueError): + filters.Derivation(text="", checks=("a", "b", "c", "d")) + + +def test_extension_matching_is_case_insensitive_and_ignores_leading_dots(): + assert filters.extension_of("Report.DOCX") == ".docx" + assert filters.extension_of(".hidden") == "" + assert filters.filter_for("Report.DOCX").name == "docx" + assert filters.filter_for("noextension") is None + + +def test_header_renders_a_list_as_a_flow_sequence_and_a_string_as_before(): + rendered = web_capture.header([("a", "plain"), ("b", ["x y", "z"]), ("c", [])]) + assert rendered == '---\na: plain\nb: ["x y", "z"]\nc: []\n---\n' + assert yaml.safe_load(rendered.strip("-\n").replace("---", "")) == {"a": "plain", "b": ["x y", "z"], "c": []} + + +# --- the nets in raw pending and raw accept --------------------------------- + + +def test_pending_names_the_derive_line_for_a_lone_file_with_a_filter(tree, capsys): + _put(tree, "report.docx", make_docx(_p("Hello"))) + _put(tree, "photo.jpg", b"\xff") + raw_pending_command(json_out=True) + by_path = {c["paths"][0]: c for c in json.loads(capsys.readouterr().out)} + assert by_path["incoming/report.docx"]["hint"] == "tools/wikitool raw derive incoming/report.docx" + assert by_path["incoming/photo.jpg"]["hint"] is None + + raw_pending_command(json_out=False) + assert "raw derive incoming/report.docx" in capsys.readouterr().out + + +def test_pending_gives_a_fetch_pair_no_hint(tree, capsys): + _put(tree, "page.html", b"

x

") + _put(tree, "page.md", b"x\n") + raw_pending_command(json_out=True) + [candidate] = json.loads(capsys.readouterr().out) + assert candidate["hint"] is None + + +def test_accept_warns_about_an_underived_file_but_still_promotes_it(tree, capsys): + path = _put(tree, "report.docx", make_docx(_p("Hello"))) + raw_accept_command( + files=[path], fidelity="verbatim", authority="reporting", + page=None, replaces=None, dry_run=False, + ) + out = capsys.readouterr().out + assert "WARN" in out and "raw derive incoming/report.docx" in out + assert [p for p in (tree / "raw").rglob("report.docx")] + + +def test_accept_is_quiet_when_a_derived_md_sits_beside_the_file(tree, capsys): + page = _put(tree, "page.html", b"

x

") + md = _put(tree, "page.md", b"x\n") + raw_accept_command( + files=[page, md], fidelity="published", authority="reporting", + page=None, replaces=None, dry_run=False, + ) + assert "WARN" not in capsys.readouterr().out + + +def test_the_web_page_filter_derives_a_saved_html_file(tree): + _put(tree, "seite.html", b"T

Kopf

Text

") + _derive("incoming/seite.html") + header, text = _header_and_text(tree / "incoming/seite/derived.md") + assert header["filter"] == "html" and header["title"] == "T" + assert header["charset"].startswith("utf-8") + assert "# Kopf" in text diff --git a/tools/chemenu/web_capture.py b/tools/chemenu/web_capture.py index 374992d..dc6e684 100644 --- a/tools/chemenu/web_capture.py +++ b/tools/chemenu/web_capture.py @@ -572,8 +572,15 @@ def _scalar(value: str) -> str: return json.dumps(value, ensure_ascii=False) -def header(fields: list[tuple[str, str]]) -> str: - lines = ["---"] + [f"{key}: {_scalar(value)}" for key, value in fields] + ["---"] +def _value(value: "str | list[str]") -> str: + """A list becomes a YAML flow sequence (a JSON array is valid YAML).""" + if isinstance(value, list): + return json.dumps(value, ensure_ascii=False) + return _scalar(value) + + +def header(fields: "list[tuple[str, str | list[str]]]") -> str: + lines = ["---"] + [f"{key}: {_value(value)}" for key, value in fields] + ["---"] return "\n".join(lines) + "\n"