From 99945c4841eed4c8dbfe9a90f240d600655394d9 Mon Sep 17 00:00:00 2001 From: Torben Nehmer Date: Tue, 13 Oct 2026 05:53:48 +0200 Subject: [PATCH] export: export run writes selected kb/ pages as a derived, read-only version into configured target folders (markdown and command handlers, manifest, after_publish), raw accept also refuses manifest-listed files, doctor checks the section (#214); 10.0.0-beta.13 Files changed: - CHANGES.md - VERSION - docs/how-to/README.md - docs/how-to/back-up-to-a-synced-folder.md - docs/how-to/collect-from-a-synced-folder.md - docs/how-to/export-guidelines.md - docs/how-to/export-to-a-synced-folder.md - docs/reference/commands/README.md - docs/reference/commands/exports.md - docs/reference/commands/git.md - docs/reference/commands/guideline-export.md - docs/reference/commands/instance-health.md - docs/reference/commands/raw-material-and-uploads.md - docs/reference/configuration.md - docs/reference/exit-codes.md - instructions/setup-instance.md - raw/CONTRACT.md - tools/CONTRACT.md - tools/README.md - tools/chemenu/backup.py - tools/chemenu/cli_contract.py - tools/chemenu/commands/doctor.py - tools/chemenu/commands/export_cmd.py - tools/chemenu/commands/git_publish.py - tools/chemenu/commands/raw/_common.py - tools/chemenu/commands/raw/accept.py - tools/chemenu/commands/raw/capture.py - tools/chemenu/config_shapes.py - tools/chemenu/guideline_export.py - tools/chemenu/page_export.py - tools/chemenu/reference_doc.py - tools/chemenu/repo_capture.py - tools/chemenu/tests/test_config_shapes.py - tools/chemenu/tests/test_export_run.py Co-Authored-By: Claude Code Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2 --- CHANGES.md | 3 +- VERSION | 2 +- docs/how-to/README.md | 2 + docs/how-to/back-up-to-a-synced-folder.md | 4 + docs/how-to/collect-from-a-synced-folder.md | 4 + docs/how-to/export-guidelines.md | 4 +- docs/how-to/export-to-a-synced-folder.md | 155 ++++ docs/reference/commands/README.md | 9 +- .../{guideline-export.md => exports.md} | 69 +- docs/reference/commands/git.md | 5 +- docs/reference/commands/instance-health.md | 1 + .../commands/raw-material-and-uploads.md | 4 +- docs/reference/configuration.md | 93 ++- docs/reference/exit-codes.md | 4 +- instructions/setup-instance.md | 5 + raw/CONTRACT.md | 21 +- tools/CONTRACT.md | 77 +- tools/README.md | 3 +- tools/chemenu/backup.py | 58 +- tools/chemenu/cli_contract.py | 4 +- tools/chemenu/commands/doctor.py | 26 + tools/chemenu/commands/export_cmd.py | 132 ++- tools/chemenu/commands/git_publish.py | 66 +- tools/chemenu/commands/raw/_common.py | 36 +- tools/chemenu/commands/raw/accept.py | 6 +- tools/chemenu/commands/raw/capture.py | 2 +- tools/chemenu/config_shapes.py | 151 +++- tools/chemenu/guideline_export.py | 60 +- tools/chemenu/page_export.py | 631 +++++++++++++++ tools/chemenu/reference_doc.py | 1 + tools/chemenu/repo_capture.py | 8 +- tools/chemenu/tests/test_config_shapes.py | 2 +- tools/chemenu/tests/test_export_run.py | 764 ++++++++++++++++++ 33 files changed, 2316 insertions(+), 96 deletions(-) create mode 100644 docs/how-to/export-to-a-synced-folder.md rename docs/reference/commands/{guideline-export.md => exports.md} (50%) create mode 100644 tools/chemenu/page_export.py create mode 100644 tools/chemenu/tests/test_export_run.py diff --git a/CHANGES.md b/CHANGES.md index af5b3ae..40d879e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,7 +14,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. --- -## 10.0.0-beta.12 - 2026-10-12 - AGENTS.md § Gates and mcp-read-server name every intake channel behind the Upload Review Gate, not only submit (#213 close) +## 10.0.0-beta.13 - 2026-10-12 - Exporte als konfigurierte Ziele: export run schreibt eine abgeleitete, schreibgeschützte Lesefassung von kb/ je Zielordner (Markdown oder über ein Konvertierprogramm), auf Wunsch nach jedem publish; raw accept lehnt auch Dateien ab, die in einem Export-Manifest stehen **Author:** Torben Nehmer @@ -37,6 +37,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. - raw fetch hinter einem Login: raw login speichert die Session des Nutzers als Secret, raw fetch holt damit; gerenderte Seiten über die optionale Browser-Komponente, externe Abrufer pro Host - Gesprächssprache: AGENTS.md § Conversation language legt fest, dass der Agent im Chat die KB-Sprache spricht; instructions sync schreibt den Sprachwert als generierte Zeile in jedes veröffentlichte SKILL.md, instructions verify und doctor melden eine geänderte Sprache als Drift - Backup in einen Sync-Ordner (OneDrive): backup init legt ein Bare-Repo unter /git als origin an, backup run und publish spiegeln die Maschinendaten nach //, doctor warnt vor .git im Sync-Ordner +- Exporte als konfigurierte Ziele: export run schreibt eine abgeleitete, schreibgeschützte Lesefassung von kb/ je Zielordner (Markdown oder über ein Konvertierprogramm), auf Wunsch nach jedem publish; raw accept lehnt auch Dateien ab, die in einem Export-Manifest stehen **Low impact** - Ketten von Paketen: stack-dev schreibt ## Kette, stack-close nennt das nächste Glied diff --git a/VERSION b/VERSION index b833795..bb590df 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -10.0.0-beta.12 +10.0.0-beta.13 diff --git a/docs/how-to/README.md b/docs/how-to/README.md index 77712b4..d680f94 100644 --- a/docs/how-to/README.md +++ b/docs/how-to/README.md @@ -15,6 +15,8 @@ Working with the wiki: browser, or through a fetch program of your own - [Capture documentation from a git repository](capture-repository-docs.md) - and keep it current - [Export guidelines to captured repositories](export-guidelines.md) +- [Export to a synced folder](export-to-a-synced-folder.md) - a read-only version of selected pages + for a reader outside the repository, OneDrive and Microsoft 365 included - [Query the wiki](query-the-wiki.md) - asking, and searching yourself - [Maintain the wiki](maintain-the-wiki.md) - lint, provenance, and the gates on the way to a publish - [Run the weekly review](run-the-weekly-review.md) - commitments in a task tracker against the diff --git a/docs/how-to/back-up-to-a-synced-folder.md b/docs/how-to/back-up-to-a-synced-folder.md index 8016171..f9ef5c2 100644 --- a/docs/how-to/back-up-to-a-synced-folder.md +++ b/docs/how-to/back-up-to-a-synced-folder.md @@ -156,3 +156,7 @@ Only one device publishes to `/git`. Two devices pushing into the same s writers again, only one step removed - and the sync client resolves that with conflict copies, not with a merge. An instance on a second device gets a target of its own. The sync client is a backup here, not a way to work on one instance from two devices. + +A folder the sync client uploads can also hold a read-only version of selected pages, for readers +outside the repository: [Export pages to a synced folder](export-to-a-synced-folder.md). It is a +separate folder with its own settings, and never the backup target. diff --git a/docs/how-to/collect-from-a-synced-folder.md b/docs/how-to/collect-from-a-synced-folder.md index bd08a3d..b1343a5 100644 --- a/docs/how-to/collect-from-a-synced-folder.md +++ b/docs/how-to/collect-from-a-synced-folder.md @@ -131,3 +131,7 @@ exits 0 with "nothing to collect". The agent does the same at the start of an ingest when you name no file: `upload collect`, then `upload list`, then a report of what waits. It never runs `upload accept` for you. + +The other direction - a read-only version of selected pages written out to a synced folder - is +[Export pages to a synced folder](export-to-a-synced-folder.md). An export is never a source: the +export marker keeps it out of `raw/`. diff --git a/docs/how-to/export-guidelines.md b/docs/how-to/export-guidelines.md index c389c13..6ebfcb2 100644 --- a/docs/how-to/export-guidelines.md +++ b/docs/how-to/export-guidelines.md @@ -50,5 +50,7 @@ did not see. ## Related -- [Guideline export commands](../reference/commands/guideline-export.md) +- [Export commands](../reference/commands/exports.md) +- [Export pages to a synced folder](export-to-a-synced-folder.md) - the other export, a read-only + version of `kb/` pages - [Why the gates are code](../explanation/why-gates-are-code.md) diff --git a/docs/how-to/export-to-a-synced-folder.md b/docs/how-to/export-to-a-synced-folder.md new file mode 100644 index 0000000..70260fe --- /dev/null +++ b/docs/how-to/export-to-a-synced-folder.md @@ -0,0 +1,155 @@ +# Export to a synced folder + +Give a reader outside the repository a read-only version of part of `kb/`: the selected pages, +written as files into a folder - typically one a sync client such as OneDrive uploads, so that a +Microsoft 365 tenant can index them. The export is derived. It is regenerated from the committed +pages on every run, nobody edits it, and nothing in it flows back into the wiki. What the settings +look like: [configuration reference](../reference/configuration.md#export-to-a-folder). + + +## Contents + +- [Pick the pages and the format](#pick-the-pages-and-the-format) +- [Set it up](#set-it-up) +- [What a run does to the folder](#what-a-run-does-to-the-folder) +- [Export after every publish](#export-after-every-publish) +- [Convert the pages with a program of your own](#convert-the-pages-with-a-program-of-your-own) +- [What does not come back in](#what-does-not-come-back-in) +- [Related](#related) + + +## Pick the pages and the format + +- **Which pages.** Only what a target's `include` selects is written; there is no "everything". A + selection is a list of `search` predicates that must all hold - `tags=shared`, `kind=system`, + `!draft` - and a page that fits any selection of the target is exported. Try a selection with + `tools/wikitool search --field tags=shared` first. +- **Which format.** Two ways, equally valid: + - `markdown` writes the pages as `.md` files. It needs nothing but the folder. + - `command` hands those files to a converter of your own and exports what it produces - Word + documents with `pandoc`, say. + + Whether a Microsoft 365 tenant indexes `.md` files for Copilot is not settled for every tenant. + Try a small `markdown` target first; if the tenant does not pick the files up, the same pages go + through a `command` target as a format it does. + +## Set it up + +1. **Pick the folder**: a folder inside the synced folder, for example + `C:\Users\\OneDrive - \Chemenu-Lesefassung`. It must be empty or not exist yet, + and it must lie outside the instance, outside every other export target, outside every `intake` + folder and outside `backup.target`. +2. **Write the `export` section** into `.wikitool-local.json` at the root of the instance - by hand, + since its fields decide what leaves this machine and which program runs, and `config set` + refuses them: + + ```json + { + "export": { + "targets": { + "shared": { + "handler": "markdown", + "path": "C:\\Users\\\\OneDrive - \\Chemenu-Lesefassung", + "include": [["tags=shared"]], + "frontmatter": "summary", + "links": "relative", + "after_publish": false + } + } + } + } + ``` + + `frontmatter` decides what follows the export marker in each file: `minimal` nothing, `summary` + (the default) the title, type, summary and modified date, `full` the page's whole frontmatter. + `links: relative` (the default) turns a link to an exported page into a relative Markdown link; + `links: text` leaves the page's name. A link to a page that is not exported is always plain text. +3. **Tell the agent:** + + > Export the pages. + + The agent runs `tools/wikitool export run`. It reports per target which pages are new, which + changed and which were removed, and how many stayed as they were. Name a target to run only that + one: `tools/wikitool export run shared`. +4. **Check it:** `tools/wikitool doctor`. The `export` line lists the targets; a `FAIL` there names + what is wrong with the section. + +The export reads the committed pages. A `kb/` or `types/` with uncommitted changes is refused, so +every file names the commit its pages come from. Publish first. + +## What a run does to the folder + +The folder holds the pages and one file of the stack's own, `.wikitool-export.json`, which lists +what the last run wrote. + +- **Only its own files.** A run removes a file only when it is listed and no longer exported, and a + folder only when the run created it. Anything else in the folder stays as it is. +- **Nobody edits an export.** A listed file that someone changed stops the target before anything is + written, and the message names the file. Move or delete it and run again. A listed file that is + missing is written again. +- **A folder that is not an export is refused.** A folder that holds files and no manifest is + somebody's folder; the run writes nothing into it. +- **An empty selection is an error.** A run that finds no page does not delete the earlier export. +- **An unchanged export stays untouched.** A run that has nothing to change writes nothing, not even + a modification time - the sync client has nothing to upload. +- **A growing selection needs no confirmation.** New pages are reported as new; read the list + before the sync client uploads them. + +Targets are independent: one that fails is reported with its reason and the others still run; the +command then exits 1. + +## Export after every publish + +With `"after_publish": true` on a target, every `tools/wikitool publish` that committed refreshes +it afterwards - with `--no-push` too, since the export reads the commit, not the remote. If that +export fails, `publish` prints a `WARN` with the reason and still exits 0: the commit and the push +stand. Fix what the line names and run `tools/wikitool export run`; another `publish` does not +repeat it. A malformed `export` section stops `publish` before it commits, but only when some target +sets `after_publish`. + +## Convert the pages with a program of your own + +A target with `"handler": "command"` names an entry of `commands`: + +```json +{ + "export": { + "targets": { + "as-word": { + "handler": "command", + "command": "pandoc-docx", + "path": "C:\\Users\\\\OneDrive - \\Chemenu-Word", + "include": [["tags=shared"]] + } + }, + "commands": { + "pandoc-docx": ["pwsh", "-NoProfile", "-File", "C:\\tools\\pandoc-docx.ps1"] + } + } +} +``` + +The pages are rendered into a staging directory first. The program runs without a shell, with stdin +closed and a limit of 300 seconds, and gets two more arguments: the directory with the Markdown +files and an empty directory for its output. On exit 0 the regular files it left there (below any +subdirectory, symlinks left out and reported) become the content of the target; the manifest and +all the rules above apply to them as to Markdown. Any other end - a non-zero exit, a timeout, a +program that does not start - leaves the target exactly as it was and prints the end of the +program's stderr. The staging directory is always removed. + +## What does not come back in + +An export is never a source. Every Markdown file starts with a frontmatter block whose first key is +`wikitool_export`, and `tools/wikitool raw accept` and `raw capture` refuse a file that carries it. +A converter may lose that key, so `raw accept` also refuses any file in `incoming/` that is byte +for byte one of the files in the manifest of a configured target - and warns, without refusing, +when a target has no manifest yet or the `export` section is broken. + +## Related + +- [Configuration reference](../reference/configuration.md#export-to-a-folder) +- [Export commands](../reference/commands/exports.md) +- [Export guidelines to captured repositories](export-guidelines.md) - the other export, one file + pushed into a repository +- [Back up to a synced folder](back-up-to-a-synced-folder.md) - the same kind of folder, for the + backup instead diff --git a/docs/reference/commands/README.md b/docs/reference/commands/README.md index dbf8f99..285cab7 100644 --- a/docs/reference/commands/README.md +++ b/docs/reference/commands/README.md @@ -15,7 +15,7 @@ collected in [exit codes and gates](../exit-codes.md). - [Provenance](#provenance) - [Raw material and uploads](#raw-material-and-uploads) - [Git](#git) -- [Guideline export](#guideline-export) +- [Exports](#exports) - [Workshop runs and session budget](#workshop-runs-and-session-budget) - [Types, instructions and docs](#types-instructions-and-docs) - [Telemetry](#telemetry) @@ -101,11 +101,12 @@ Full records: [git.md](git.md) - [`backup init`](git.md#backup-init) - Create the backup target's bare repository and make it `origin` when there is none. - [`backup run`](git.md#backup-run) - Mirror this machine's files that are not in git into `//`. -## Guideline export +## Exports -Full records: [guideline-export.md](guideline-export.md) +Full records: [exports.md](exports.md) -- [`export guidelines`](guideline-export.md#export-guidelines) - Render the guideline pages into one generated `GUIDELINES.md`, or push it into the captured repositories that opted in (**Guideline Push Gate**). +- [`export guidelines`](exports.md#export-guidelines) - Render the guideline pages into one generated `GUIDELINES.md`, or push it into the captured repositories that opted in (**Guideline Push Gate**). +- [`export run`](exports.md#export-run) - Write the derived, read-only reading copy of `kb/` into the folders the `export` section configures - the selected pages as Markdown, or converted by a program of this checkout. ## Workshop runs and session budget diff --git a/docs/reference/commands/guideline-export.md b/docs/reference/commands/exports.md similarity index 50% rename from docs/reference/commands/guideline-export.md rename to docs/reference/commands/exports.md index d8c4415..ffbe763 100644 --- a/docs/reference/commands/guideline-export.md +++ b/docs/reference/commands/exports.md @@ -1,6 +1,13 @@ -# Guideline export +# Exports -Writing the generated `GUIDELINES.md` into the repositories the wiki has captured. +Derived, read-only versions of `kb/`: the generated `GUIDELINES.md` for the repositories the wiki has captured, and the pages exported into configured folders. + + +## Contents + +- [`export guidelines`](#export-guidelines) +- [`export run`](#export-run) + ## `export guidelines` @@ -74,4 +81,62 @@ tools/wikitool export guidelines --confirm --push --field tags=guideline - `wikitool search` - the same predicates, to preview the selection - `wikitool raw capture` - how a repository becomes a target - `instructions/gates.md` - the gate procedure + +## `export run` + +Write the derived, read-only reading copy of `kb/` into the folders the `export` section configures - the selected pages as Markdown, or converted by a program of this checkout. + +**Synopsis** + +- `wikitool export run [ ...]` - Every configured target, or the named ones. Needs an `export` section in `.wikitool-local.json`; the user writes it by hand. + +**Options** + +```text +Arguments: + [TARGET]... Targets to run; default: every configured target. + +Options: + -h, --help Show this message and exit. +``` + +**Examples** + +```bash +tools/wikitool export run +tools/wikitool export run work-notes +``` + +**Behaviour** + +- An export is derived: regenerated from `kb/` on every run, never edited, never read back into the wiki. Each target names its pages with an allowlist of `search --field` predicates (`include`: a list of selections, AND within one, OR across them), a handler (`markdown`, or `command` for the operator's converter) and the folder (`path`, outside the instance). Nothing is exported that no selection names. +- Every Markdown file starts with a frontmatter block whose `wikitool_export` line says `kind=pages instance= commit=`, followed by `title`, `type`, `summary`, `modified` where the page has them (`frontmatter: summary`, the default; `minimal` keeps only the marker, `full` the whole frontmatter), then the page text. A file keeps its path below `kb/`. A wikilink to an exported page becomes a relative Markdown link (`links: relative`, the default) or plain text (`links: text`); a link to a page that is not exported is always plain text. Footnote markers stay; code is never rewritten. LF endings, no timestamp: the same pages give the same bytes. +- Handler `command`: the Markdown is rendered into a staging directory, the program of `export.commands` runs with no shell, stdin closed and a timeout of 300 s, and receives the Markdown directory and an empty output directory as its last two arguments. On exit 0 the regular files in the output directory (recursively) become the target's content; any other end leaves the target unchanged and prints the end of its stderr. The staging directory is always removed. +- The folder is only ever touched by what it holds in its manifest, `.wikitool-export.json`: the files an earlier run wrote, each with its sha256. A run removes only manifest files that are no longer exported (and folders it created, once empty), rewrites a manifest file that is missing, leaves every other file byte for byte as it was, and writes nothing - not even a modification time - for a file that did not change. A run that changes nothing writes nothing at all. +- Refuses a target, before writing anything into it: a non-empty folder with no manifest (it is somebody's, not an export), a manifest file that was changed in the folder since the last run (naming it), a foreign file where an exported one belongs, an empty selection (it would delete the earlier export), an unknown field in a predicate. A missing folder is created. +- Refuses everything, before any target: uncommitted changes in `kb/` or `types/` (untracked files included), no git checkout, a page under `kb/` with unreadable frontmatter, an unknown target name, a broken `export` section. +- Output per target: `new` (with the page titles), `changed`, `removed`, the count `unchanged`, or `failed` with the reason. A page that newly enters a selection is reported as new, not gated. Exit 0 when every target ran; exit 1 when any failed - the others were still served. +- A target with `after_publish: true` is also refreshed by every `publish` that committed, `--no-push` included; a failure there is a warning, the commit stands, and this command is the repeat. +- `raw accept` refuses a file whose sha256 a target's manifest lists, and one that carries the marker: an export never goes back into `raw/`. + +**Properties** + +- Changes files: yes +- Idempotent: yes +- Atomic: Per file within a target (a temporary file in the target, then a move over its place), the manifest last; targets are independent, so a failed target does not undo another +- Counts toward the session call budget: yes +- Can reach the network: no + +**Exit codes** + +- `0` - success +- `1` - No `export` section, an unknown target name, or a broken section (a predicate that does not parse, a path inside the instance or in the way of another target, an `intake` folder or the backup target) +- `1` - `kb/` or `types/` has uncommitted changes, the instance is not a git checkout, or a page under `kb/` has unreadable frontmatter +- `1` - A target failed: a foreign folder, a file changed in the target, an empty selection, an unknown field, a converter that did not exit 0 + +**See also** + +- `wikitool search` - the same predicates, to preview a selection +- `wikitool backup run` - the same kind of folder, for this machine's own files +- `docs/how-to/export-to-a-synced-folder.md` diff --git a/docs/reference/commands/git.md b/docs/reference/commands/git.md index b14926c..5060d24 100644 --- a/docs/reference/commands/git.md +++ b/docs/reference/commands/git.md @@ -134,6 +134,7 @@ tools/wikitool publish --confirm-rebase --message "ingest: docker-cheats - `--path` (repeatable) scopes the whole operation - gate count, staging and commit - to that subtree, with one exception: uncommitted generated files outside `--path` (`kb/index.md`, `kb/log.md`, `kb/provenance.md`, every `INDEX.md` under `kb/`) are staged and committed too, and a line names them - whether a reconcile or the change itself wrote them. They stay out of the gate count, and the `--confirm` token and the re-run line cover the `--path` given. Every other file outside `--path` is left as it is. - Commit message: `--message`, then a `Files changed:` paragraph listing every committed path. When `--message` ends in a paragraph git reads as a trailer block (`Co-Authored-By:` and the like), that block stays the last paragraph and the list goes in front of it, since git reads trailers from the last paragraph only. `git interpret-trailers` decides whether there is such a block, and the list moves only when git reads the same trailers from the result; otherwise it is appended at the end. `--message` is not part of any gate token. - Machine-data backup: with a `backup` section in `.wikitool-local.json` whose `machine_data` is true, a call that pushed then mirrors this machine's files that are not in git into `//` - the same step as `backup run`. A failure there prints a `WARN` line with the reason and still exits 0, since the commit and the push stand. No backup with `--no-push`, after a failed push, or when there was nothing to push. A malformed `backup` section stops the call with exit 1 before anything is committed, `--no-push` included. +- Export after publish: every target of the `export` section with `after_publish: true` is refreshed once a commit was made - the same step as `export run`, `--no-push` included, since the export reads the committed `kb/`. A failure there prints a `WARN` line with the reason and still exits 0, since the commit stands; `export run` repeats it. A malformed `export` section stops the call with exit 1 before anything is committed, but only when some target sets `after_publish`. - After a successful commit or push whose changed files include `tools/`, `types/`, `instructions/`, `AGENTS.md` or a path ending in `CONTRACT.md`, prints one reminder line: the phase past this point (an issue-body rewrite, `docs/` staleness, a changelog entry's accuracy) is not covered by `docs verify`, `instructions verify` or `pytest`. It is not a gate: no exit code change, nothing to clear, and silent for an ordinary content publish. **Properties** @@ -149,11 +150,12 @@ tools/wikitool publish --confirm-rebase --message "ingest: docker-cheats - `0` - success - `0` - The machine-data backup after the push failed: a `WARN` line names the reason, and the commit and the push stand +- `0` - The export after the commit failed: a `WARN` line names the target and the reason, and the commit stands - `1` - git failed - `git add`, `git commit`, `git push`, or the reconcile's automatic rebase (a conflict outside the generated files, an edited `kb/log.md`, or an uncommitted change on a path the incoming commits change) - `1` - The push target (`--branch`) is not the checked-out branch, or HEAD is detached; the unborn branch of a fresh `git init` is not this case - `1` - No `--no-push`, and the remote is not configured or cannot be reached; nothing was committed - `1` - `--yes`/`-y` was passed - the flag does not exist and fails with an explicit error -- `1` - The settings files do not pass their check, `remotes.allowed_push_urls` is not a list of strings, or the `backup` section is malformed; nothing was committed +- `1` - The settings files do not pass their check, `remotes.allowed_push_urls` is not a list of strings, the `backup` section is malformed, or the `export` section is malformed while a target sets `after_publish`; nothing was committed - `42` - The remotes settings file of an earlier version is still there - `42` - Mass-Update Gate: `--threshold` (default 10) or more counted files would be committed, or the `--confirm` token does not match this changeset - `42` - Rebase-Review Gate: `/` moved and both sides changed the same file; the output lists the upstream commits, the overlapping files and their diff @@ -165,6 +167,7 @@ tools/wikitool publish --confirm-rebase --message "ingest: docker-cheats - `instructions/gates.md` - the gate procedure - `instructions/setup-instance.md` - the first publish of a new instance - `wikitool backup run` - the machine-data backup on its own +- `wikitool export run` - the page export on its own ## `backup init` diff --git a/docs/reference/commands/instance-health.md b/docs/reference/commands/instance-health.md index 425087b..dd012f0 100644 --- a/docs/reference/commands/instance-health.md +++ b/docs/reference/commands/instance-health.md @@ -38,6 +38,7 @@ tools/wikitool doctor --json - Legacy settings (`legacy-settings`): one `FAIL` for each per-feature settings file of an earlier version (telemetry, remotes, upload, task tracker) that is still there. Nothing reads them any more, so the settings they hold are not in effect; the fix is `tools/wikitool config migrate`. - Synced folder (`sync-folder`): a `WARN` when the git directory, links and junctions resolved, lies under a folder named by `OneDrive`, `OneDriveCommercial` or `OneDriveConsumer` (compared without regard to case on Windows) or under `~/Library/CloudStorage/` (macOS); `OK` otherwise, never a `FAIL`. - Backup (`backup`): the `backup` section absent is `OK`, malformed is a `FAIL`. Configured, a `WARN` when `/git` is not a bare repository, `origin` does not push there, or - with `machine_data` true - no machine-data backup exists yet; otherwise `OK` with the age of the last machine-data backup. +- Page export (`export`): the `export` section absent is `OK`, malformed is a `FAIL`; otherwise `OK` listing each target with its handler, folder and whether it runs after `publish`. - Intake channels (`upload-intake`): the channels of the `intake` section with their handler, and how many submissions wait in `quarantine/`. Absent is `OK` and means no channel - no `submit` tool, nothing to collect; malformed is a `FAIL`, and so is an `mcp-upload/` of an earlier version that still holds submissions (the message names the command that moves it). A `folder` channel whose `path` does not exist is a `WARN`. - Task tracker: the `tasks` section present, absent or malformed - absent is `OK` (no tracker configured), malformed is a `FAIL`. When `WIKITOOL_TASKS_CONFIG` is set, the profile it names is read instead of the `tasks`, `superproductivity` and `caldav` sections and the finding names it; a set variable that names no file is a `FAIL`, never `OK`. - For a configured `superproductivity` provider, the configured `access` path's own state: `access: "api"` reports whether its local REST API answers `GET /health` with a ready renderer right now, `access: "snapshot"` whether a backup file is ready. The other access path is never attempted, and neither state is ever a `FAIL`. diff --git a/docs/reference/commands/raw-material-and-uploads.md b/docs/reference/commands/raw-material-and-uploads.md index 96c9a9d..c445cb6 100644 --- a/docs/reference/commands/raw-material-and-uploads.md +++ b/docs/reference/commands/raw-material-and-uploads.md @@ -262,7 +262,7 @@ tools/wikitool raw capture --update raw/2026/10/service-docs - `--ref` is a branch name (`main`) or a tag pattern (`v*`, any of `*?[`), which names the newest matching tag by version order. It resolves to one commit - for an annotated tag, the commit it points at. - The commit is fetched by ref name, shallowly, into a bare cache repository per URL under `tools/.wikitool_capture/` (gitignored). Files are read from it as blobs, never through a checkout, so they are byte-identical to the repository's - no line-ending conversion, no filter. - `--path` globs (repeatable) select files by their path in the repository, as git's `:(glob)` pathspec does: `*`, `?` and `[...]` do not cross `/`; `**` as a whole segment spans any number of directories; a glob without wildcards also takes everything below it as a directory. Files keep their repository paths inside the bundle. -- Never captured, each named in the output with its reason: a file whose first line starts with ` @@ -93,7 +94,7 @@ Each file is a JSON object of sections, one per feature, beside two keys of its the id; the id names the directory of the secrets file below. The sections are [telemetry](#telemetry), [remotes](#allowed-push-urls), [tasks, superproductivity and caldav](#task-tracker) and [intake](#intake-channels), [sites](#site-handlers), [fetch](#fetch-routes) and -[backup](#backup-to-a-synced-folder). `config set` names the sections it knows, and `doctor` reports a +[backup](#backup-to-a-synced-folder) and [export](#export-to-a-folder). `config set` names the sections it knows, and `doctor` reports a section it does not read as a `WARN`. **One place per value.** A value in the file its tier does not name is an error naming the file, @@ -631,6 +632,96 @@ Example: `config set` exits 42 for them, and only you write them. The procedure, restoring included: [back-up-to-a-synced-folder.md](../how-to/back-up-to-a-synced-folder.md). +## Export to a folder + +The `export` section of `.wikitool-local.json` declares folders that hold a derived, read-only +version of `kb/` - for a reader outside the repository, such as a Microsoft 365 tenant that indexes +a OneDrive folder. An export is regenerated from the committed pages on every run and is never +edited; what someone changes in the folder is refused or overwritten, never merged back. Each +target has a handler: `markdown` writes the selected pages as Markdown files, `command` renders +them into a staging directory and hands that to a converter of this checkout. + + +| Field | Tier | Type | Required | Default | Description | +|-------|------|------|----------|---------|-------------| +| `targets` | checkout, gate | object | yes | - | Target name (lower-case letters, digits and hyphens) to target; the name is the argument of `export run`. Every target has `handler` (`markdown`: the pages as Markdown files; `command`: the same Markdown run through a converter of this checkout), `path` (an absolute path outside the instance; it must not lie in or contain another target, an `intake` folder or `backup.target`) and `include` (an array of selections, each an array of `search --field` predicates: all predicates of a selection must hold, and a page that fits any selection is exported; there is no "everything"). Optional: `frontmatter` (`minimal`: only the export marker; `summary`, the default: also `title`, `type`, `summary` and `modified` where the page has them; `full`: the page's whole frontmatter), `links` (`relative`, the default: a wikilink to an exported page becomes a relative Markdown link; `text`: plain text; a link to a page that is not exported is always plain text) and `after_publish` (default false: refresh this target after every `publish` that committed). The handler `command` takes `command` (required: the name of an entry of `commands`). A field the handler does not take is an error. | +| `commands` | checkout, names a command | object | no | `{}` | Command name to command: an array of the program and its arguments, run by the `command` handler without a shell, with stdin closed and a timeout of 300 s. The last two arguments appended are the directory holding the Markdown and an empty output directory. Exit 0 makes the regular files in the output directory the target's content; anything else leaves the target unchanged and prints the end of the program's stderr. | + +- **Absent:** No export: `export run` stops with exit 1 naming the section, `publish` exports nothing and `doctor` reports `OK`. +- **Malformed:** An error. `export run` stops with exit 1 and `doctor` reports `FAIL`; `publish` stops with exit 1 before it commits anything, but only when some target sets `after_publish`. A broken section is never read as "no export". + +Example: + +```json +{ + "export": { + "targets": { + "work-notes": { + "handler": "markdown", + "path": "C:\\Users\\\\OneDrive - \\Chemenu-Lesefassung", + "include": [ + [ + "kind=system" + ], + [ + "tags=shared", + "!draft" + ] + ], + "frontmatter": "summary", + "links": "relative", + "after_publish": true + }, + "as-word": { + "handler": "command", + "command": "pandoc-docx", + "path": "C:\\Users\\\\OneDrive - \\Chemenu-Word", + "include": [ + [ + "kind=system" + ] + ] + } + }, + "commands": { + "pandoc-docx": [ + "pwsh", + "-NoProfile", + "-File", + "C:\\tools\\pandoc-docx.ps1" + ] + } + } +} +``` + + +Within one target, `include` is a list of selections, each a list of `search` predicates (`tags=docs`, +`entity_type=system`, `!source_url`): a page is exported when it satisfies every predicate of at least +one selection. Only what `include` selects is written - there is no way to exclude by default. The +syntax is checked when the section is read, the field names against the corpus when `export run` +runs. `path` must not lie in or contain the instance, another target, an `intake` folder or +`backup.target`. + +Every Markdown file starts with a frontmatter block whose first key is `wikitool_export`, carrying +the kind, the instance and the commit of the newest selected page. `frontmatter` decides what follows: +`minimal` nothing, `summary` the title, type, summary and modified date, `full` the page's whole +frontmatter. Wikilinks between exported pages become relative Markdown links (`links: relative`) or +plain text (`links: text`); a link to a page that is not exported is always plain text. Footnote +markers and code stay as they are. + +A target folder holds `.wikitool-export.json`, which lists the files a run wrote with their digests. +A run removes only listed files that are no longer exported, creates only folders it needs and +removes only those it created, and leaves every other file untouched. It stops before writing for a +listed file someone changed, for a folder that is not empty and has no manifest, and for a selection +that comes up empty. A run that finds nothing to change writes nothing, not even a modification time. + +`targets` and `commands` decide what leaves this checkout and which program runs, so they are marked +`gate` and `names a command`: `config set` exits 42 for them, and only you write them. `raw accept` +also refuses a file whose sha256 is listed in the manifest of a configured target, for an export +that went through a converter and lost its marker. The procedure: +[export-to-a-synced-folder.md](../how-to/export-to-a-synced-folder.md). + ## Files of earlier versions Until version 10 every feature above had a file of its own at the root: `.wikitool-telemetry.json`, diff --git a/docs/reference/exit-codes.md b/docs/reference/exit-codes.md index ea462a2..2e2cd08 100644 --- a/docs/reference/exit-codes.md +++ b/docs/reference/exit-codes.md @@ -10,7 +10,7 @@ What each exit code means, and how a caller reacts to it, is [AGENTS.md § Tool ### guideline-push -- [`export guidelines`](commands/guideline-export.md#export-guidelines) +- [`export guidelines`](commands/exports.md#export-guidelines) ### mass-update @@ -45,6 +45,6 @@ What each exit code means, and how a caller reacts to it, is [AGENTS.md § Tool - [`publish`](commands/git.md#publish): Mass-Update Gate: `--threshold` (default 10) or more counted files would be committed, or the `--confirm` token does not match this changeset - [`publish`](commands/git.md#publish): Rebase-Review Gate: `/` moved and both sides changed the same file; the output lists the upstream commits, the overlapping files and their diff - [`publish`](commands/git.md#publish): Publish-Remote Gate: `remotes.allowed_push_urls` is set and the push URL of `--remote` is not listed in it, or `--remote` resolves to no push URL -- [`export guidelines`](commands/guideline-export.md#export-guidelines): Guideline Push Gate: `--push` without `--confirm`, or with a token that does not match the current set of targets +- [`export guidelines`](commands/exports.md#export-guidelines): Guideline Push Gate: `--push` without `--confirm`, or with a token that does not match the current set of targets - [`config set`](commands/configuration.md#config-set): The field names a program to run or affects a gate diff --git a/instructions/setup-instance.md b/instructions/setup-instance.md index 72a1324..1dc5b2f 100644 --- a/instructions/setup-instance.md +++ b/instructions/setup-instance.md @@ -130,6 +130,11 @@ one it needs before that. show the user any `remotes.allowed_push_urls` entry it prints, for them to add. The whole procedure is [docs/how-to/back-up-to-a-synced-folder.md](../docs/how-to/back-up-to-a-synced-folder.md). Until it is done, treat the instance as local. + - A reader outside the repository - a Microsoft 365 tenant that indexes a OneDrive folder, say + - who should get a read-only version of part of `kb/`: offer an export. The user writes the + `export` section of `.wikitool-local.json` by hand (its fields are gates, so `config set` + refuses them); then run `tools/wikitool export run`. The whole procedure is + [docs/how-to/export-to-a-synced-folder.md](../docs/how-to/export-to-a-synced-folder.md). 4. **Decision point - authoring conventions.** The release ships no filled-in conventions, only `kb/CONVENTIONS.md.template` and one `kb//COLLECTION.md.template` per collection. Both diff --git a/raw/CONTRACT.md b/raw/CONTRACT.md index d07de9e..1974490 100644 --- a/raw/CONTRACT.md +++ b/raw/CONTRACT.md @@ -95,12 +95,16 @@ That is what keeps the clean-up safe: only directories the moves emptied are rem can go with them. A file *inside* a subdirectory is never accepted on its own; the refusal names both ways out - the whole folder, or the file moved up into `incoming/`. -**A guideline export never comes back in.** A file whose first line (after an optional BOM) -starts with ` +## Footnotes + +[^s-one]: [[Source - One]] + +""" + +BETA = """ +# Beta + +Back to [[Alpha]]. +""" + + +def _git(root: Path, *args: str) -> str: + result = subprocess.run(["git", *args], cwd=root, capture_output=True, text=True) + assert result.returncode == 0, result.stderr + return result.stdout.strip() + + +def _commit(root: Path, message: str = "change") -> str: + _git(root, "add", "-A") + _git(root, "commit", "-q", "-m", message) + return _git(root, "rev-parse", "HEAD") + + +def _page(root: Path, rel: str, body: str, **frontmatter) -> Path: + path = root / "kb" / rel + meta = {"type": "types/concept.md", "tags": ["docs"], "summary": f"About {path.stem}.", + "modified": "2026-10-01", "related": ["Beta"], **frontmatter} + write_page(path, meta, body) + return path + + +@pytest.fixture +def instance(tmp_path, monkeypatch): + root = tmp_path / "instance" + root.mkdir() + _git(root, "init", "-q", "-b", "main") + _git(root, "config", "user.name", "Test") + _git(root, "config", "user.email", "test@example.com") + (root / "incoming").mkdir() + (root / "raw").mkdir() + (root / "incoming" / ".gitkeep").write_text("", encoding="utf-8") + (root / ".gitignore").write_text( + "/incoming/*\n!/incoming/.gitkeep\n/.wikitool-local.json\n/quarantine/\n/reports/\n", + encoding="utf-8") + shutil.copytree(config._PACKAGE_ROOT / "types", root / "types") + monkeypatch.setattr(config, "TYPES_DIR", root / "types") + monkeypatch.setattr(resolver, "_repo_root", root) + _page(root, "concepts/Alpha.md", ALPHA) + _page(root, "concepts/protocols/Beta.md", BETA) + _page(root, "concepts/Gamma.md", "\n# Gamma\n\nPrivate.\n", tags=["private"]) + _commit(root, "init") + (tmp_path / "OneDrive - Example").mkdir() + monkeypatch.setattr(config, "ROOT", root) + monkeypatch.setenv("WIKITOOL_SESSION_ID", "test-session") + return root + + +def _out(root: Path, name: str = "Pages") -> Path: + return root.parent / "OneDrive - Example" / name + + +def _configure(root: Path, **targets) -> None: + """One target `pages` by default, with `docs` pages selected.""" + if not targets: + targets = {"pages": {}} + section = {} + for name, extra in targets.items(): + body = {"handler": "markdown", "path": str(_out(root, name)), "include": [["tags=docs"]]} + body.update(extra) + section[name] = body + commands = {body["command"]: ["unset"] for body in section.values() if body.get("command")} + export = {"targets": section, **({"commands": commands} if commands else {})} + (root / config.LOCAL_SETTINGS_FILENAME).write_text( + json.dumps({"schema": 1, "export": export}), encoding="utf-8") + + +def _set_commands(root: Path, commands: dict) -> None: + path = root / config.LOCAL_SETTINGS_FILENAME + data = json.loads(path.read_text(encoding="utf-8")) + data["export"]["commands"] = commands + path.write_text(json.dumps(data), encoding="utf-8") + + +def _run(*names: str): + return runner.invoke(app, ["export", "run", *names]) + + +def _flat(text: str) -> str: + return " ".join(text.split()) + + +def _snapshot(directory: Path) -> dict: + return {p.relative_to(directory).as_posix(): p.read_bytes() + for p in sorted(directory.rglob("*")) if p.is_file()} + + +def _mtimes(directory: Path) -> dict: + return {p.relative_to(directory).as_posix(): p.stat().st_mtime_ns + for p in sorted(directory.rglob("*")) if p.is_file()} + + +def _manifest(folder: Path) -> dict: + return json.loads((folder / page_export.MANIFEST_NAME).read_text(encoding="utf-8")) + + +# --- the section ------------------------------------------------------------- + + +def test_no_section_is_an_error_naming_it_and_doctor_says_ok(instance): + result = _run() + assert result.exit_code == 1 + assert "no export section" in _flat(result.output) + assert _check("export").status == "OK" + + +@pytest.mark.parametrize("target, needle", [ + ({"path": "relative/folder"}, "absolute"), + ({"include": []}, "include"), + ({"include": [[]]}, "include"), + ({"include": [["not a predicate"]]}, "predicate"), + ({"handler": "pdf"}, "handler"), + ({"frontmatter": "everything"}, "frontmatter"), + ({"links": "hyper"}, "links"), + ({"after_publish": "yes"}, "after_publish"), + ({"command": "convert"}, "command"), + ({"handler": "command"}, "command"), +]) +def test_a_broken_section_stops_run_and_doctor_fails(instance, target, needle): + _configure(instance, pages=target) + result = _run() + assert result.exit_code == 1, result.output + assert needle in _flat(result.output) + assert _check("export").status == "FAIL" + assert not _out(instance, "pages").exists() + + +def test_a_target_inside_the_instance_is_refused(instance): + _configure(instance, pages={"path": str(instance / "site")}) + assert _run().exit_code == 1 + assert _check("export").status == "FAIL" + + +def test_targets_lying_in_one_another_are_refused(instance): + _configure(instance, one={"path": str(_out(instance, "one"))}, + two={"path": str(_out(instance, "one") / "inner")}) + result = _run() + assert result.exit_code == 1 + assert "lie in one another" in _flat(result.output) + + +def test_a_target_in_the_backup_target_or_an_intake_folder_is_refused(instance): + backup_target = _out(instance, "backup") + path = instance / config.LOCAL_SETTINGS_FILENAME + _configure(instance, pages={"path": str(backup_target / "pages")}) + data = json.loads(path.read_text(encoding="utf-8")) + data["backup"] = {"target": str(backup_target)} + path.write_text(json.dumps(data), encoding="utf-8") + assert _run().exit_code == 1 + data.pop("backup") + data["intake"] = {"channels": {"inbox": { + "handler": "folder", "path": str(_out(instance, "pages") / "in"), "max_bytes": 1024, + "allowed_extensions": [".md"]}}} + data["export"]["targets"]["pages"]["path"] = str(_out(instance, "pages")) + path.write_text(json.dumps(data), encoding="utf-8") + assert _run().exit_code == 1 + + +def test_the_operator_only_fields_cannot_be_set_by_the_agent(): + export = config_shapes.EXPORT + assert export.field("targets").user_only and export.field("commands").user_only + assert export.field("targets").tier == config_shapes.CHECKOUT + assert export.field("commands").tier == config_shapes.CHECKOUT + + +def test_an_unknown_target_name_is_an_error_and_changes_nothing(instance): + _configure(instance) + result = _run("pages", "nothing") + assert result.exit_code == 1 + assert "unknown export target" in _flat(result.output) + assert not _out(instance, "pages").exists() + + +# --- what a run writes ------------------------------------------------------- + + +def test_a_run_writes_the_selected_pages_with_marker_and_links(instance): + _configure(instance) + result = _run() + assert result.exit_code == 0, result.output + folder = _out(instance, "pages") + assert set(_snapshot(folder)) == { + "concepts/Alpha.md", "concepts/protocols/Beta.md", page_export.MANIFEST_NAME} + commit = _git(instance, "rev-parse", "HEAD") + alpha = (folder / "concepts" / "Alpha.md").read_text(encoding="utf-8") + lines = alpha.split("\n") + assert lines[0] == "---" + assert lines[1] == f'wikitool_export: "kind=pages instance=local commit={commit}"' + assert "title: Alpha" in alpha and "summary: About Alpha." in alpha + assert "tags" not in alpha and "related" not in alpha + assert "[Beta](protocols/Beta.md)" in alpha and "[the beta page](protocols/Beta.md)" in alpha + assert "also [Beta](protocols/Beta.md)" in alpha + assert "and Gamma for more" in alpha + assert "Cited[^s-one]." in alpha and "[^s-one]: " in alpha + assert "Inline `[[Beta]]` stays." in alpha + assert "```\n[[Beta]] and [[Gamma]]\n```" in alpha + assert alpha.endswith("\n") and not alpha.endswith("\n\n") and "\r" not in alpha + beta = (folder / "concepts" / "protocols" / "Beta.md").read_text(encoding="utf-8") + assert "[Alpha](../Alpha.md)" in beta + assert guideline_export.file_is_export(folder / "concepts" / "Alpha.md") + out = _flat(result.output) + assert "new (2)" in out and "Alpha" in out and "unchanged: 0" in out + + +def test_frontmatter_modes_and_links_text(instance): + _configure(instance, minimal={"frontmatter": "minimal", "links": "text"}, + full={"frontmatter": "full"}) + assert _run().exit_code == 0 + minimal = (_out(instance, "minimal") / "concepts" / "Alpha.md").read_text(encoding="utf-8") + assert minimal.startswith('---\nwikitool_export: "kind=pages') + assert minimal.split("---\n")[1].count("\n") == 1 + assert "[Beta](" not in minimal and "See Beta, the beta page and Gamma" in minimal + full = (_out(instance, "full") / "concepts" / "Alpha.md").read_text(encoding="utf-8") + assert "tags:" in full and "type: types/concept.md" in full and "related:" in full + + +def test_selections_are_or_and_predicates_are_and(instance): + _configure(instance, pages={"include": [["tags=docs", "summary~Alpha"], ["tags=private"]]}) + assert _run().exit_code == 0 + assert set(_snapshot(_out(instance, "pages"))) == { + "concepts/Alpha.md", "concepts/Gamma.md", page_export.MANIFEST_NAME} + + +def test_an_unknown_field_is_an_error_at_run(instance): + _configure(instance, pages={"include": [["no_such_field=x"]]}) + result = _run() + assert result.exit_code == 1 + assert "unknown field" in _flat(result.output) + assert not _out(instance, "pages").exists() + + +def test_a_rerun_on_an_unchanged_corpus_touches_nothing(instance): + _configure(instance) + assert _run().exit_code == 0 + folder = _out(instance, "pages") + before, stamps = _snapshot(folder), _mtimes(folder) + result = _run() + assert result.exit_code == 0 + assert _snapshot(folder) == before and _mtimes(folder) == stamps + assert "unchanged: 2" in _flat(result.output) + assert "new (" not in result.output and "changed (" not in result.output + + +def test_an_instance_commit_that_touches_no_selected_page_changes_nothing(instance): + _configure(instance) + assert _run().exit_code == 0 + folder = _out(instance, "pages") + stamps = _mtimes(folder) + (instance / "README.md").write_text("other\n", encoding="utf-8") + _commit(instance, "unrelated") + assert _run().exit_code == 0 + assert _mtimes(folder) == stamps + + +def test_a_changed_page_a_new_page_and_a_removed_page(instance): + _configure(instance) + assert _run().exit_code == 0 + folder = _out(instance, "pages") + _page(instance, "concepts/Alpha.md", ALPHA + "\nMore.\n") + _page(instance, "concepts/Delta.md", "\n# Delta\n\nNew.\n") + _page(instance, "concepts/protocols/Beta.md", BETA, tags=["private"]) + commit = _commit(instance, "edit") + result = _run() + assert result.exit_code == 0, result.output + out = _flat(result.output) + assert "new (1): Delta" in out and "changed (1):" in out and "removed (1):" in out + assert (folder / "concepts" / "Delta.md").is_file() + assert not (folder / "concepts" / "protocols" / "Beta.md").exists() + assert not (folder / "concepts" / "protocols").exists() + assert "More." in (folder / "concepts" / "Alpha.md").read_text(encoding="utf-8") + assert _manifest(folder)["commit"] == commit + assert [e["path"] for e in _manifest(folder)["files"]] == ["concepts/Alpha.md", "concepts/Delta.md"] + + +def test_the_selection_growing_needs_no_gate(instance): + _configure(instance) + assert _run().exit_code == 0 + for index in range(30): + _page(instance, f"concepts/Many{index}.md", f"\n# Many{index}\n\nText.\n") + _commit(instance, "many") + result = _run() + assert result.exit_code == 0, result.output + assert "new (30)" in _flat(result.output) + + +def test_foreign_files_stay_and_only_created_directories_are_pruned(instance): + _configure(instance) + assert _run().exit_code == 0 + folder = _out(instance, "pages") + (folder / "notes.txt").write_bytes(b"mine\r\n") + (folder / "concepts" / "mine.txt").write_bytes(b"also mine") + (folder / "concepts" / "protocols" / "mine.txt").write_bytes(b"in a created dir") + _page(instance, "concepts/protocols/Beta.md", BETA, tags=["private"]) + _commit(instance, "drop beta") + assert _run().exit_code == 0 + assert (folder / "notes.txt").read_bytes() == b"mine\r\n" + assert (folder / "concepts" / "mine.txt").read_bytes() == b"also mine" + assert (folder / "concepts" / "protocols" / "mine.txt").read_bytes() == b"in a created dir" + assert (folder / "concepts" / "Alpha.md").is_file() + + +def test_a_directory_that_existed_before_the_export_is_kept(instance): + _page(instance, "concepts/protocols/Beta.md", BETA, tags=["private"]) + _commit(instance, "no beta yet") + _configure(instance) + assert _run().exit_code == 0 + folder = _out(instance, "pages") + (folder / "concepts" / "protocols").mkdir() + _page(instance, "concepts/protocols/Beta.md", BETA) + _commit(instance, "beta") + assert _run().exit_code == 0 + assert "concepts/protocols" not in _manifest(folder)["dirs"] + _page(instance, "concepts/protocols/Beta.md", BETA, tags=["private"]) + _commit(instance, "drop beta") + assert _run().exit_code == 0 + assert (folder / "concepts" / "protocols").is_dir() + + +def test_a_non_empty_folder_without_a_manifest_is_refused(instance): + folder = _out(instance, "pages") + folder.mkdir() + (folder / "mine.md").write_text("mine\n", encoding="utf-8") + _configure(instance) + result = _run() + assert result.exit_code == 1 + assert "not empty" in _flat(result.output) and "mine.md" not in _snapshot(folder).keys() - {"mine.md"} + assert _snapshot(folder) == {"mine.md": b"mine\n"} + + +def test_an_empty_existing_folder_is_taken(instance): + _out(instance, "pages").mkdir() + _configure(instance) + assert _run().exit_code == 0 + + +def test_a_file_changed_by_hand_aborts_the_target_before_any_write(instance): + _configure(instance) + assert _run().exit_code == 0 + folder = _out(instance, "pages") + edited = folder / "concepts" / "protocols" / "Beta.md" + edited.write_text("my own words\n", encoding="utf-8") + _page(instance, "concepts/Alpha.md", ALPHA + "\nMore.\n") + _commit(instance, "edit") + before = _snapshot(folder) + result = _run() + assert result.exit_code == 1 + assert "concepts/protocols/Beta.md" in _flat(result.output) + assert _snapshot(folder) == before + + +def test_a_missing_manifest_file_is_rewritten(instance): + _configure(instance) + assert _run().exit_code == 0 + folder = _out(instance, "pages") + expected = _snapshot(folder) + (folder / "concepts" / "Alpha.md").unlink() + result = _run() + assert result.exit_code == 0, result.output + assert _snapshot(folder) == expected + + +def test_a_crash_between_the_files_and_the_manifest_is_recovered(instance): + _configure(instance) + assert _run().exit_code == 0 + folder = _out(instance, "pages") + old_manifest = (folder / page_export.MANIFEST_NAME).read_bytes() + _page(instance, "concepts/Alpha.md", ALPHA + "\nMore.\n") + _commit(instance, "edit") + assert _run().exit_code == 0 + new_state = _snapshot(folder) + (folder / page_export.MANIFEST_NAME).write_bytes(old_manifest) + assert _run().exit_code == 0 + assert _snapshot(folder) == new_state + + +def test_zero_selected_pages_is_an_error_and_deletes_nothing(instance): + _configure(instance) + assert _run().exit_code == 0 + folder = _out(instance, "pages") + before = _snapshot(folder) + _configure(instance, pages={"include": [["tags=nothing-has-this"]]}) + result = _run() + assert result.exit_code == 1 + assert "no page matches" in _flat(result.output) + assert _snapshot(folder) == before + + +def test_a_dirty_kb_is_refused(instance): + _configure(instance) + _page(instance, "concepts/Alpha.md", ALPHA + "\nunsaved\n") + result = _run() + assert result.exit_code == 1 + assert "uncommitted" in _flat(result.output) + assert not _out(instance, "pages").exists() + + +def test_a_page_with_unreadable_frontmatter_stops_the_run(instance): + _configure(instance) + (instance / "kb" / "concepts" / "Broken.md").write_text("---\ntype: [unclosed\n---\nBody\n", encoding="utf-8") + _commit(instance, "broken") + result = _run() + assert result.exit_code == 1 + assert "Broken.md" in result.output + assert not _out(instance, "pages").exists() + + +def test_targets_are_independent_and_any_failure_is_exit_1(instance): + bad = _out(instance, "bad") + bad.mkdir() + (bad / "mine.md").write_text("mine\n", encoding="utf-8") + _configure(instance, bad={}, good={}) + result = _run() + assert result.exit_code == 1 + assert (_out(instance, "good") / "concepts" / "Alpha.md").is_file() + assert _snapshot(bad) == {"mine.md": b"mine\n"} + assert "bad" in result.output and "failed" in result.output + + +def test_named_targets_run_alone(instance): + _configure(instance, one={}, two={}) + assert _run("two").exit_code == 0 + assert _out(instance, "two").is_dir() and not _out(instance, "one").exists() + + +def test_a_case_only_collision_is_refused(instance): + _page(instance, "concepts/alpha.md", "\n# alpha\n\nLower.\n") + _commit(instance, "case") + _configure(instance) + result = _run() + assert result.exit_code == 1 + assert "differ only in case" in _flat(result.output) + assert not _out(instance, "pages").exists() + + +# --- the command handler ----------------------------------------------------- + + +def _converter(root: Path, body: str) -> list[str]: + script = root.parent / "convert.py" + script.write_text(textwrap.dedent(body), encoding="utf-8") + return [sys.executable, str(script)] + + +def _command_target(root: Path, program: list[str]) -> None: + _configure(root, pages={"handler": "command", "command": "convert"}) + _set_commands(root, {"convert": program}) + + +COPY_AS_TXT = """ + import pathlib, sys + src, out = map(pathlib.Path, sys.argv[1:3]) + for f in src.rglob("*.md"): + dest = out / f.relative_to(src).with_suffix(".txt") + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(f.read_text(encoding="utf-8").upper(), encoding="utf-8") +""" + + +def test_the_command_handler_exports_what_the_program_leaves(instance): + _command_target(instance, _converter(instance, COPY_AS_TXT)) + result = _run() + assert result.exit_code == 0, result.output + folder = _out(instance, "pages") + assert set(_snapshot(folder)) == { + "concepts/Alpha.txt", "concepts/protocols/Beta.txt", page_export.MANIFEST_NAME} + assert "WIKITOOL_EXPORT" in (folder / "concepts" / "Alpha.txt").read_text(encoding="utf-8") + stamps = _mtimes(folder) + assert _run().exit_code == 0 and _mtimes(folder) == stamps + + +def test_the_command_gets_two_directories_no_stdin_and_the_staging_is_removed(instance, tmp_path): + record = tmp_path / "record.json" + program = _converter(instance, f""" + import json, os, pathlib, sys + src, out = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]) + stdin = sys.stdin.read() + pathlib.Path({str(record)!r}).write_text(json.dumps({{ + "args": sys.argv[1:], "stdin": stdin, "cwd": os.getcwd(), + "src_files": sorted(p.name for p in src.rglob("*.md")), "out_files": os.listdir(out)}})) + (out / "all.txt").write_text("done") + """) + _command_target(instance, program) + assert _run().exit_code == 0 + seen = json.loads(record.read_text(encoding="utf-8")) + assert seen["stdin"] == "" and seen["out_files"] == [] and seen["src_files"] == ["Alpha.md", "Beta.md"] + assert Path(seen["args"][0]).name == "markdown" and Path(seen["args"][1]).name == "output" + assert not Path(seen["args"][0]).parent.exists() + + +def test_a_failing_command_leaves_the_target_as_it_was(instance): + _command_target(instance, _converter(instance, COPY_AS_TXT)) + assert _run().exit_code == 0 + folder = _out(instance, "pages") + before = _snapshot(folder) + _page(instance, "concepts/Alpha.md", ALPHA + "\nMore.\n") + _commit(instance, "edit") + _set_commands(instance, {"convert": _converter(instance, """ + import sys + sys.stderr.write("x" * 5000 + "boom at the end") + sys.exit(3) + """)}) + result = _run() + assert result.exit_code == 1 + text = _flat(result.output) + assert "exit 3" in text and "boom at the end" in text and "x" * 3000 not in text + assert _snapshot(folder) == before + + +def test_a_missing_program_and_a_timeout_are_target_errors(instance, monkeypatch): + _command_target(instance, ["/nonexistent/converter"]) + result = _run() + assert result.exit_code == 1 and "could not be started" in _flat(result.output) + _set_commands(instance, {"convert": _converter(instance, "import time; time.sleep(30)")}) + monkeypatch.setattr(page_export, "COMMAND_TIMEOUT_SECONDS", 1) + result = _run() + assert result.exit_code == 1 and "did not finish" in _flat(result.output) + assert not _out(instance, "pages").exists() + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks") +def test_symlinks_in_the_output_are_skipped_and_reported(instance): + _command_target(instance, _converter(instance, """ + import os, pathlib, sys + out = pathlib.Path(sys.argv[2]) + (out / "real.txt").write_text("real") + os.symlink("/etc/hostname", out / "link.txt") + """)) + result = _run() + assert result.exit_code == 0, result.output + assert set(_snapshot(_out(instance, "pages"))) == {"real.txt", page_export.MANIFEST_NAME} + assert "link.txt" in result.output + + +def test_a_converter_that_leaves_nothing_is_an_error(instance): + _command_target(instance, _converter(instance, "pass")) + assert _run().exit_code == 1 + assert not _out(instance, "pages").exists() + + +# --- publish ----------------------------------------------------------------- + + +def _publish(**overrides): + kwargs = dict(message="change", push=False, confirm=None, confirm_rebase=None, yes=False, + threshold=10, remote="origin", branch="main", path=None) + kwargs.update(overrides) + publish_command(**kwargs) + + +def test_publish_exports_after_the_commit_with_no_push_too(instance): + _configure(instance, pages={"after_publish": True}, manual={}) + _page(instance, "concepts/Alpha.md", ALPHA + "\nMore.\n") + _publish() + folder = _out(instance, "pages") + head = _git(instance, "rev-parse", "HEAD") + assert _manifest(folder)["commit"] == head + assert "More." in (folder / "concepts" / "Alpha.md").read_text(encoding="utf-8") + assert not _out(instance, "manual").exists() + + +def test_a_failed_export_after_publish_is_a_warning_and_the_commit_stands(instance, capsys): + folder = _out(instance, "pages") + folder.mkdir() + (folder / "mine.md").write_text("mine\n", encoding="utf-8") + _configure(instance, pages={"after_publish": True}) + head = _git(instance, "rev-parse", "HEAD") + _page(instance, "concepts/Alpha.md", ALPHA + "\nMore.\n") + _publish() + out = capsys.readouterr().out + assert "WARN" in out and "export run" in out + assert _git(instance, "rev-parse", "HEAD") != head + + +def test_publish_without_after_publish_exports_nothing(instance): + _configure(instance) + _page(instance, "concepts/Alpha.md", ALPHA + "\nMore.\n") + _publish() + assert not _out(instance, "pages").exists() + + +def test_a_malformed_section_stops_publish_only_when_a_target_asks_for_it(instance): + _configure(instance, pages={"after_publish": True, "include": []}) + _page(instance, "concepts/Alpha.md", ALPHA + "\nMore.\n") + head = _git(instance, "rev-parse", "HEAD") + with pytest.raises(typer.Exit) as exc: + _publish() + assert exc.value.exit_code == 1 and _git(instance, "rev-parse", "HEAD") == head + _configure(instance, pages={"include": []}) + _publish() + assert _git(instance, "rev-parse", "HEAD") != head + + +# --- doctor ------------------------------------------------------------------ + + +def _check(name: str): + assert name == "export" + return doctor.check_export() + + +def test_doctor_lists_the_targets(instance): + _configure(instance, pages={"after_publish": True}) + check = _check("export") + assert check.status == "OK" + assert "pages" in check.detail and "after publish" in check.detail + + +# --- raw accept -------------------------------------------------------------- + + +def _accept(path: Path): + return runner.invoke(app, ["raw", "accept", str(path), "--fidelity", "verbatim", + "--authority", "normative"]) + + +def test_raw_accept_refuses_a_file_carrying_the_page_export_marker(instance): + _configure(instance) + assert _run().exit_code == 0 + exported = (_out(instance, "pages") / "concepts" / "Alpha.md").read_bytes() + (instance / "incoming" / "Alpha.md").write_bytes(exported) + result = _accept(instance / "incoming" / "Alpha.md") + assert result.exit_code == 1 + assert "export" in _flat(result.output) and "nothing was moved" in _flat(result.output) + assert (instance / "incoming" / "Alpha.md").is_file() + assert not any((instance / "raw").rglob("*.md")) + + +def test_raw_accept_refuses_an_exported_file_whose_marker_a_converter_dropped(instance): + _command_target(instance, _converter(instance, COPY_AS_TXT)) + assert _run().exit_code == 0 + exported = (_out(instance, "pages") / "concepts" / "Alpha.txt").read_bytes() + folder = instance / "incoming" / "pack" + folder.mkdir() + (folder / "note.txt").write_text("ordinary\n", encoding="utf-8") + (folder / "Alpha.txt").write_bytes(exported) + (instance / "incoming" / "Alpha.txt").write_bytes(exported) + for target in (folder, instance / "incoming" / "Alpha.txt"): + before = _snapshot(instance / "incoming") + result = _accept(target) + assert result.exit_code == 1, result.output + assert "configured export target" in _flat(result.output) + assert "Alpha.txt" in result.output + assert _snapshot(instance / "incoming") == before + assert not any(p.is_file() for p in (instance / "raw").rglob("*")) + + +def test_raw_accept_warns_and_goes_on_when_a_manifest_is_missing(instance): + _configure(instance) + (instance / "incoming" / "note.md").write_text("# Note\n\nOrdinary.\n", encoding="utf-8") + result = _accept(instance / "incoming" / "note.md") + assert "WARN" in result.output and "no manifest" in _flat(result.output) + assert "configured export target" not in result.output + + +def test_raw_accept_warns_and_goes_on_when_the_section_is_broken(instance): + _configure(instance, pages={"include": []}) + (instance / "incoming" / "note.md").write_text("# Note\n\nOrdinary.\n", encoding="utf-8") + result = _accept(instance / "incoming" / "note.md") + assert "WARN" in result.output and "export section is broken" in _flat(result.output) + assert "configured export target" not in result.output + + +def test_known_digests_lists_the_manifest_files(instance): + _configure(instance) + assert _run().exit_code == 0 + digests, warnings = page_export.known_digests() + alpha = (_out(instance, "pages") / "concepts" / "Alpha.md").read_bytes() + assert hashlib.sha256(alpha).hexdigest() in digests and warnings == [] + + +# --- the marker -------------------------------------------------------------- + + +FRONT = b'---\nwikitool_export: "kind=pages instance=local commit=abc"\ntitle: A\n---\n\nBody\n' + + +@pytest.mark.parametrize("data, expected", [ + (b"\nbody", True), + (FRONT, True), + (b"\xef\xbb\xbf" + FRONT, True), + (FRONT.replace(b"\n", b"\r\n"), True), + (b"---\ntitle: A\n---\nwikitool_export: x\n", False), + (b"---\ntitle: A\nwikitool_export: x\n", False), + (b"# Title\n---\nwikitool_export: x\n---\n", False), + (b"---\ntitle: A\n---\n\nBody\n", False), + (b"", False), +]) +def test_is_export_recognises_both_forms(data, expected): + assert guideline_export.is_export(data) is expected + + +def test_the_key_beyond_the_scan_window_is_not_seen(): + padding = b"x: " + b"y" * 100 + b"\n" + data = b"---\n" + padding * (guideline_export.SCAN_BYTES // len(padding) + 2) + b"wikitool_export: z\n---\n" + assert guideline_export.is_export(data) is False + + +def test_file_is_export_reads_files(tmp_path): + (tmp_path / "a.md").write_bytes(FRONT) + (tmp_path / "b.md").write_bytes(b"plain\n") + assert guideline_export.file_is_export(tmp_path / "a.md") + assert not guideline_export.file_is_export(tmp_path / "b.md") + + +def test_the_marker_key_is_defined_once(): + package = Path(guideline_export.__file__).parent + definitions = [ + path for path in package.rglob("*.py") + if "tests" not in path.parts and "wikitool_export:" in path.read_text("utf-8") + and 'b"wikitool_export:"' in path.read_text("utf-8") + ] + assert definitions == [Path(guideline_export.__file__)]