From 893db6e6403a95d45f5e98077e2c988adec5051e Mon Sep 17 00:00:00 2001 From: Torben Nehmer Date: Wed, 7 Oct 2026 08:44:50 +0200 Subject: [PATCH] feat: the shapes of the four hand-written .wikitool-*.json files live in one record each (config_shapes.py); loaders, doctor and docs/reference/configuration.md read them (#193) The telemetry, remotes, upload and tasks loaders take field names, defaults, coercions and their Expected: text from the record and accept exactly what they did before. doctor WARNs on a telemetry file the policy silently reads differently. configuration.md carries one generated region per file, also checked in an instance. docs reference --apply now refreshes several regions in one page in sequence instead of keeping only the last. Files changed: - CHANGES.md - VERSION - docs/how-to/set-up-the-mcp-server.md - docs/reference/README.md - docs/reference/commands/instance-health.md - docs/reference/commands/types-instructions-and-docs.md - docs/reference/configuration.md - instructions/dev/doc-pull-through.md - instructions/setup-instance.md - tools/CONTRACT.md - tools/README.md - tools/chemenu/commands/docs_verify.py - tools/chemenu/commands/doctor.py - tools/chemenu/commands/git_publish.py - tools/chemenu/config.py - tools/chemenu/config_shapes.py - tools/chemenu/reference_doc.py - tools/chemenu/tasks/caldav.py - tools/chemenu/tasks/config.py - tools/chemenu/tasks/superproductivity.py - tools/chemenu/telemetry/policy.py - tools/chemenu/tests/test_config_shapes.py - tools/chemenu/tests/test_doctor.py - tools/chemenu/tests/test_reference_doc.py - tools/chemenu/upload.py Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2 --- CHANGES.md | 23 +- VERSION | 2 +- docs/how-to/set-up-the-mcp-server.md | 18 +- docs/reference/README.md | 11 +- docs/reference/commands/instance-health.md | 2 +- .../commands/types-instructions-and-docs.md | 3 +- docs/reference/configuration.md | 241 +++++-- instructions/dev/doc-pull-through.md | 2 +- instructions/setup-instance.md | 5 +- tools/CONTRACT.md | 5 +- tools/README.md | 3 +- tools/chemenu/commands/docs_verify.py | 24 +- tools/chemenu/commands/doctor.py | 42 +- tools/chemenu/commands/git_publish.py | 13 +- tools/chemenu/config.py | 15 +- tools/chemenu/config_shapes.py | 596 +++++++++++++++++ tools/chemenu/reference_doc.py | 31 +- tools/chemenu/tasks/caldav.py | 52 +- tools/chemenu/tasks/config.py | 68 +- tools/chemenu/tasks/superproductivity.py | 66 +- tools/chemenu/telemetry/policy.py | 24 +- tools/chemenu/tests/test_config_shapes.py | 608 ++++++++++++++++++ tools/chemenu/tests/test_doctor.py | 43 ++ tools/chemenu/tests/test_reference_doc.py | 8 + tools/chemenu/upload.py | 52 +- 25 files changed, 1646 insertions(+), 311 deletions(-) create mode 100644 tools/chemenu/config_shapes.py create mode 100644 tools/chemenu/tests/test_config_shapes.py diff --git a/CHANGES.md b/CHANGES.md index 797db0f..2cd069a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,7 +14,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. --- -## 9.0.0-beta.7 - 2026-10-07 - INSTALL.md is an English installation tutorial; new tutorial from a first source to a first answer with a shipped sample; README.md a landing page +## 9.0.0-beta.8 - 2026-10-07 - Die Formen der vier handgeschriebenen .wikitool-*.json stehen je in einem Datensatz (config_shapes), aus dem Lader, doctor und die Konfigurationsreferenz lesen **Author:** Torben Nehmer @@ -32,6 +32,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. - docs reference: skill catalogue and page types under docs/reference/, generated from the shipped defaults in the origin repo only - docs/: how-to guides, configuration reference and two explanation pages carved out of README.md, INSTALL.md, INSTALL-MCP.md and EVALS.md; the last two retired - INSTALL.md is an English installation tutorial; new tutorial from a first source to a first answer with a shipped sample; README.md a landing page +- Die Formen der vier handgeschriebenen .wikitool-*.json stehen je in einem Datensatz (config_shapes), aus dem Lader, doctor und die Konfigurationsreferenz lesen **Low impact** - version bump: Abschlussmeldung nennt Stichpunkt oder Themenabsatz statt pauschal Prosa @@ -45,7 +46,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. ### The root documents split into `docs/` pages (#190) -The task and explanation parts of `README.md`, `INSTALL.md`, `INSTALL-MCP.md` and `EVALS.md` are now single English pages: eleven how-to guides under `docs/how-to/` (ingest, capture, guideline export, query, maintenance, weekly review, Obsidian, updating an instance, troubleshooting, the MCP read server, tracing and scoring sessions), a hand-written `docs/reference/configuration.md`, and `docs/explanation/architecture.md` and `telemetry-and-evaluation.md`. A guide says what to prepare, what to tell the agent and where it stops; it links an agent's procedure instead of retelling it. `INSTALL-MCP.md` and `EVALS.md` are gone from the root and from `dist export`, so an upgrade removes them as retired; `README.md` is a landing page, and `INSTALL.md` keeps only the installation itself. Pointers in messages and command records, among them `version check`'s, name the new pages. +The task and explanation parts of `README.md`, `INSTALL.md`, `INSTALL-MCP.md` and `EVALS.md` are now single English pages: eleven how-to guides under `docs/how-to/` (ingest, capture, guideline export, query, maintenance, weekly review, Obsidian, updating an instance, troubleshooting, the MCP read server, tracing and scoring sessions), `docs/reference/configuration.md`, and `docs/explanation/architecture.md` and `telemetry-and-evaluation.md`. A guide says what to prepare, what to tell the agent and where it stops; it links an agent's procedure instead of retelling it. `INSTALL-MCP.md` and `EVALS.md` are gone from the root and from `dist export`, so an upgrade removes them as retired; `README.md` is a landing page, and `INSTALL.md` keeps only the installation itself. Pointers in messages and command records, among them `version check`'s, name the new pages. ### Doku-Gerüst nach Diátaxis unter `docs/` (#187) @@ -68,16 +69,16 @@ Migrationsdokument auf die Basis des Kandidaten, schreibt jeder `version bump` d grenzüberschreitenden Eintrag, der sein Dokument nicht nennt. Die Formregeln stehen nur noch in `instructions/dev/version-parts.md`. -### Generierte Referenz für Menschen unter `docs/reference/` (#188, #189) +### Generierte Referenz für Menschen unter `docs/reference/` (#188, #189, #193) -`wikitool docs reference [--apply]` erzeugt aus den Kommando-Datensätzen eine Seite je Gruppe unter -`docs/reference/commands/` samt Übersicht und `exit-codes.md` mit Gates und Exit-42-Ursachen, ohne -die an Agenten gerichteten Reaktionen und NEVER-Zeilen. Dazu kommen `skills.md`, der Katalog der -ausgelieferten Skills aus ihrer `description`, und `page-types.md` mit Subtypen, Zielverzeichnissen -und Pflichtfeldern je `root: kb`-Type-Spec. Diese beiden beschreiben die ausgelieferten Vorgaben -und werden nur im Ursprungs-Repo erzeugt und geprüft; in einer Instanz lässt `docs reference` sie -unberührt. Titel und Kopfsatz schreibt ein Mensch. `docs verify` meldet veraltete Regionen und -fehlende Seiten, solange es `docs/` gibt. +`wikitool docs reference [--apply]` erzeugt aus den Kommando-Datensätzen eine Seite je Gruppe samt +Übersicht und `exit-codes.md`, ohne die an Agenten gerichteten Reaktionen. `skills.md` und +`page-types.md` beschreiben die ausgelieferten Vorgaben und entstehen nur im Ursprungs-Repo. Die +Formen von `.wikitool-telemetry.json`, `-remotes`, `-upload` und `-tasks` stehen je in einem +Datensatz in `config_shapes.py`; die Lader lesen daraus Felder, Defaults und `Expected:`-Texte bei +unveränderter Akzeptanz, und `configuration.md` trägt je Datei eine generierte Region, auch in einer +Instanz. `doctor` warnt bei einer Telemetrie-Datei, die still anders gelesen wird. `docs verify` +meldet veraltete Regionen. --- diff --git a/VERSION b/VERSION index 2ea9c76..db200ba 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -9.0.0-beta.7 +9.0.0-beta.8 diff --git a/docs/how-to/set-up-the-mcp-server.md b/docs/how-to/set-up-the-mcp-server.md index 0a6ceb3..2616979 100644 --- a/docs/how-to/set-up-the-mcp-server.md +++ b/docs/how-to/set-up-the-mcp-server.md @@ -161,21 +161,9 @@ reviewed yet. ## Optional: the submit path Without this, `submit` does not exist as a tool - not unused, but not registered. A file -`.wikitool-upload.json` in the served corpus turns it on: - -```json -{ - "schema": 1, - "identity_header": "X-Forwarded-User", - "max_bytes": 10485760, - "allowed_extensions": [".md", ".txt", ".pdf", ".html", ".csv", ".json", ".png", ".jpg"], - "quota": { "submissions_per_day": 20, "bytes_per_day": 52428800 } -} -``` - -Every field is required, and none has a built-in default except `identity_header`. A malformed file -stops the server from starting. It never means "no limits": the failure deliberately goes the safe -way. Size and quota are your deployment's decision, not defaults worth copying blindly. +`.wikitool-upload.json` in the served corpus turns it on. Its fields, defaults and what a malformed +file does are in the [configuration reference](../reference/configuration.md#mcp-submissions); a +broken file never means "no limits", it stops the server from starting. **With `submit` on, the middleware has a second duty.** It has to set the identity header itself, on every authenticated request, and discard any copy the client sent. The server trusts that header as diff --git a/docs/reference/README.md b/docs/reference/README.md index 6e2b35f..4887cd7 100644 --- a/docs/reference/README.md +++ b/docs/reference/README.md @@ -14,9 +14,10 @@ written by hand. - [Page types](page-types.md) - every page type the stack ships for `kb/`, its subtypes, where its pages are written and its required fields, generated from the type-specs. - [Configuration](configuration.md) - the environment variables `wikitool` reads and the - per-checkout files an operator may create. Written by hand: it has no source to be generated - from yet. + per-checkout files an operator may create. The shape of each file is generated from the record + its loader reads it with; the environment variables and what each setting does are written by + hand. -The last two describe the stack's shipped defaults, which an instance may change or extend; -`tools/wikitool instructions list` and `tools/wikitool types describe ` show what one -instance has. +The skill catalogue and the page types describe the stack's shipped defaults, which an instance +may change or extend; `tools/wikitool instructions list` and `tools/wikitool types describe ` +show what one instance has. diff --git a/docs/reference/commands/instance-health.md b/docs/reference/commands/instance-health.md index 97866f0..574b1e3 100644 --- a/docs/reference/commands/instance-health.md +++ b/docs/reference/commands/instance-health.md @@ -39,7 +39,7 @@ tools/wikitool doctor --json - For a configured `superproductivity` provider, the configured `access` path's own state: `access: "api"` reports whether its local REST API answers `GET /health` with a ready renderer right now, `access: "snapshot"` whether a backup file is ready. The other access path is never attempted, and neither state is ever a `FAIL`. - For a configured `caldav` provider, whether the server is reachable and Basic auth succeeds - never a `FAIL`; only a broken config block is. - Session id source: `OK` for `WIKITOOL_SESSION_ID` or a registered harness variable, `WARN` only for the bare parent-pid fallback. -- Telemetry: on or off and why - installation-form default, `.wikitool-telemetry.json`, or `WIKI_TRACE` - and the current session's count and byte total against both caps; never a `FAIL`. +- Telemetry: on or off and why - installation-form default, `.wikitool-telemetry.json`, or `WIKI_TRACE` - and the current session's count and byte total against both caps; never a `FAIL`. A `WARN` when `.wikitool-telemetry.json` is not read as written - unreadable JSON, not an object, `enabled` not a JSON boolean, a cap not a positive integer, or an unknown key - naming the field; telemetry itself drops or reinterprets such a value without a word, so this is where it shows. - Exits 1 only on a `FAIL`; a missing remote, session id or `VERSION` is a `WARN`, not a fault. - Read-only and exempt from the Iteration Budget Gate. diff --git a/docs/reference/commands/types-instructions-and-docs.md b/docs/reference/commands/types-instructions-and-docs.md index f086b4d..1e21834 100644 --- a/docs/reference/commands/types-instructions-and-docs.md +++ b/docs/reference/commands/types-instructions-and-docs.md @@ -288,7 +288,7 @@ tools/wikitool docs verify - Every reference file `docs toc` covers carries the current table-of-contents region for its own headings - missing and stale are one check. - Every relative markdown link in one of those reference files resolves to an existing file. A target's `#anchor` suffix is stripped first, and code fences and inline code spans are masked before scanning, so link syntax shown as an example is not mistaken for a real reference. - Every file under `docs/` sits in one of its areas - `tutorials/`, `how-to/`, `reference/`, `explanation/` or `dev/` - never directly in `docs/` or in another directory. -- If `docs/` exists: every generated region of the human reference under `docs/reference/` matches what `docs reference` would write, every `cli_contract.GROUPS` entry has its page under `docs/reference/commands/`, and no page there belongs to no group. The skill catalogue and the page types are checked only in the origin repo (no `.wikitool-release.json`): they describe the stack's shipped defaults, which an instance's own type-specs and instructions may differ from. +- If `docs/` exists: every generated region of the human reference under `docs/reference/` matches what `docs reference` would write, every `cli_contract.GROUPS` entry has its page under `docs/reference/commands/`, and no page there belongs to no group. That includes the four configuration regions in `docs/reference/configuration.md`. The skill catalogue and the page types are checked only in the origin repo (no `.wikitool-release.json`): they describe the stack's shipped defaults, which an instance's own type-specs and instructions may differ from. - `INSTALL.md` carries one generated `` region per platform value of `tools/prerequisites.txt` (`prerequisites-` for a platform-specific one), each current; and the `` markers in `instructions/setup-instance.md` and `INSTALL.md` name the same set of keys, so a question the agent asks is never one the human guide leaves out, nor the reverse. - `VERSION` parses and the newest versioned `CHANGES.md` entry names it. An entry that crosses the compatibility boundary from the last release carries `**Breaking Change:**`, has a migration document or `**Migration:** none required`, and its `**Migration:**` line names every required migration document targeting it. - The newest versioned `CHANGES.md` entry is at most 32 000 bytes (UTF-8) as `version notes` prints it, and each of its `###` sections is exactly one paragraph of at most 1 000 bytes, heading line excluded. Older entries and a changelog with no versioned entry are not checked. @@ -495,6 +495,7 @@ tools/wikitool docs reference --apply - Rewrites the generated region of every page of the human command reference from the `cli_contract` records: one `docs/reference/commands/.md` per `cli_contract.GROUPS` entry (``), the overview in `docs/reference/commands/README.md` (``), and the gates and exit-42 causes in `docs/reference/exit-codes.md` (``). - Also, in the origin repo only, the skill catalogue `docs/reference/skills.md` (``: one row per shipped `instructions//SKILL.md`, its description split at the sentence starting "Use when") and the page types `docs/reference/page-types.md` (``: one section per `root: kb` type-spec, from what `types describe` reads). Both describe the stack's shipped defaults. In an instance - a tree with `.wikitool-release.json` - they are left alone and the output says so: an instance's adopted type-specs and own instructions may differ, and rewriting a shipped page there would block the next `dist upgrade`. +- Also the shapes of the four hand-written configuration files in `docs/reference/configuration.md`, one region each (``, `-remotes`, `-tasks`, `-upload`) inside the hand-written section on that file, rendered from the records in `chemenu.config_shapes` the loaders read the files with. Those records ship with `tools/`, so these regions are written and checked in an instance as well. Several regions in one page are refreshed one after the other into the same text. - Renders each command's summary, synopsis, options, examples, notes, properties in words, exit codes with their causes and see-also - but not the ON FAILURE reactions or NEVER, which address an agent; `tools/CONTRACT.md` keeps those. - Refreshes the table of contents of each page it writes in the same step. - Never creates a page or places a region: a page's title and one-sentence head are written by hand. A group's page is named after the group - lower case, every run of other characters one hyphen. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index b019870..c611fd2 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -4,9 +4,14 @@ The environment variables `wikitool` reads, and the per-checkout files an operat root of an instance. None of them is needed for an instance to work: an instance installed from a release runs against the origin's public release feed with no configuration at all. -This page is written by hand. Unlike the command reference, it has no single source in the code it -could be generated from yet, so a change to one of these files' shape has to be carried here -by whoever makes it. +The shape of each file - its fields, what an absent or a malformed file means, an example - is +generated from the record `wikitool` itself reads the file with, so it cannot describe a field the +tool does not take. What a setting does around it is written by hand. + +Three more files at the root are written by `wikitool` and never by hand: `.wikitool-release.json`, +`.wikitool-kb.json` and `.wikitool-tools.json` ([AGENTS.md invariant 1](../../AGENTS.md#invariants)). +The last one has its own section [below](#tool-paths), because the preflight that writes it is +something you run. ## Contents @@ -47,15 +52,33 @@ publish. Only a clone of the origin repository, used to develop the stack, has n traces are the instrument the stack measures itself with, and they are **on**. To turn the default around, create `.wikitool-telemetry.json` at the root. It is per checkout and -gitignored, with no `.template`, like `.wikitool-remotes.json`: +gitignored, with no `.template`, like `.wikitool-remotes.json`. + + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `schema` | any | no | - | Accepted and not read. Conventionally `1`; no version of this file is told apart by it yet. | +| `enabled` | boolean | no | the installation-form default | Telemetry on or off. `WIKI_TRACE` beats it in both directions. Any other value is read by its truthiness, so `"no"` means on. | +| `max_session_bytes` | positive integer | no | `5242880` | Byte cap per session trace. `WIKI_TRACE_MAX_SESSION_BYTES` beats it. A number written as a string is accepted. | +| `keep_sessions` | positive integer | no | `250` | How many session directories retention keeps. `WIKI_TRACE_KEEP_SESSIONS` beats it. A number written as a string is accepted. | + +- **Absent:** The installation-form default applies: on in a clone of the origin repository, off in an instance. +- **Malformed:** Never an error. Unreadable JSON or a non-object counts as an absent file, and a cap that is not a positive integer as an absent cap. `doctor` reports each such deviation as a `WARN` under `telemetry`, naming the field. +- **Unknown keys:** Ignored; `doctor` reports them as a `WARN`. + +Example: ```json -{ "enabled": true, "max_session_bytes": 5242880, "keep_sessions": 250 } +{ + "enabled": true, + "max_session_bytes": 5242880, + "keep_sessions": 250 +} ``` + -All three keys are optional. `WIKI_TRACE` still overrides `enabled` in both directions and beats -this file. `wikitool doctor` reports the current state: on or off, why, and how much has been -recorded against both caps. What the caps do and what never reaches a trace: +`wikitool doctor` reports the current state: on or off, why, and how much has been recorded against +both caps. What the caps do and what never reaches a trace: [telemetry-and-evaluation.md](../explanation/telemetry-and-evaluation.md). ## Tool paths @@ -74,25 +97,124 @@ file by hand. `doctor` reports under `tool-paths` whether every path still works ## Allowed push URLs `.wikitool-remotes.json` at the root restricts where `publish` may push, per checkout and -gitignored: +gitignored. + + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `schema` | any | no | - | Accepted and not read. Conventionally `1`; no version of this file is told apart by it yet. | +| `allowed_push_urls` | array of strings | yes | - | The push URLs `publish` may write to, compared with the URL the remote resolves to for a push - not with its name. An empty list allows no push at all. | + +- **Absent:** Any remote is allowed; the Publish-Remote Gate does not apply. +- **Malformed:** `publish` stops with an error before committing anything. A broken file is never read as "no restriction". +- **Unknown keys:** Ignored. + +Example: ```json -{ "allowed_push_urls": ["git@example.org:me/my-wiki.git"] } +{ + "schema": 1, + "allowed_push_urls": [ + "" + ] +} ``` + -Without the file, any remote is allowed. With it, `publish` refuses with exit 42 (the Publish-Remote -Gate) before pushing to a URL not on the list. A malformed file is an error, not permission. Only you -edit this file. The gate has no token and no flag, and an agent that edits the list to get past a -refusal is opening a gate on its own initiative. +With the file, `publish` refuses with exit 42 (the Publish-Remote Gate) before pushing to a URL not +on the list. Only you edit this file. The gate has no token and no flag, and an agent that edits the +list to get past a refusal is opening a gate on its own initiative. ## Task tracker **Optional.** The weekly review (`tools/wikitool review`, skill `gtd-weekly-review`) checks the project pages under `kb/gtd/` against a task tracker. Which tracker that is goes in `.wikitool-tasks.json` at the root. Like the telemetry and remotes files, it is per checkout and has -no `.template`, and it is **gitignored as soon as a token is in it**. Without the file, no tracker is -configured, which is a valid state, not an error. A broken file, on the other hand, is a `FAIL`: an -unreadable configuration must never pass as "no tracker". +no `.template`, and it is **gitignored as soon as a token is in it**. + + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `schema` | any | no | - | Accepted and not read. Conventionally `1`; no version of this file is told apart by it yet. | +| `provider` | string | yes | - | Which adapter the review goes through. The provider's own settings sit in a section of the same name. | +| `thresholds` | object | yes | - | The weekly review's three thresholds. | +| `thresholds.stalled_waiting_days` | positive integer | yes | - | A waiting-for item older than this many days counts as overdue. A number written as a string is accepted. | +| `thresholds.unpaged_project_weeks` | positive integer | yes | - | A tracker project with no `kb/` page is reported once it is older than this many weeks. A number written as a string is accepted. | +| `thresholds.someday_stale_months` | positive integer | yes | - | A someday/maybe item untouched for this many months counts as stale. A number written as a string is accepted. | + +- **Absent:** No tracker is configured - a valid state. The weekly review has nothing to check against. +- **Malformed:** An error, and `doctor` reports `FAIL`. A broken file is never read as "no tracker configured". +- **Unknown keys:** Ignored, so the section of a provider not chosen right now can stay in the file. + +**The `superproductivity` section.** Connection settings for the Super Productivity adapter. + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `access` | string | yes | - | Which way in: `api`, the local REST API (reads the current state and carries the write path), or `snapshot`, the newest backup snapshot (read-only). | + +With `access: "api"`: + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `api_base_url` | string | no | `"http://127.0.0.1:3876"` | Where the app's REST API listens. An empty value means the default. | +| `api_token` | string | yes, not empty | - | The token from the app's settings. | + +With `access: "snapshot"`: + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `backups_dir` | string | yes, not empty | - | The directory the app writes its backup snapshots to; `~` is expanded. | + +- **Unknown keys:** Rejected: a section names only one access path's own fields, so a `backups_dir` beside `access: "api"` is an error, not ignored. + +```json +{ + "superproductivity": { + "access": "api", + "api_base_url": "http://127.0.0.1:3876", + "api_token": "..." + } +} +``` + +```json +{ + "superproductivity": { + "access": "snapshot", + "backups_dir": "~/.config/superProductivity/backups" + } +} +``` + +**The `caldav` section.** Connection settings for the CalDAV adapter. + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `url` | string | yes, not empty | - | The account's CalDAV calendar home set. A trailing `/` is added when missing. | +| `username` | string | yes, not empty | - | The login name, which can differ from the user id in the URL. | +| `app_password` | string | yes, not empty | - | An app password is recommended over the account password. | +| `inbox_list` | string | yes, not empty | - | The name of the one inbox list. | +| `someday_list` | string | yes, not empty | - | The name of the one someday/maybe list. | +| `exclude_lists` | array of strings | no | `[]` | Existing plain task lists that are not projects. | + +- **Unknown keys:** Rejected. + +```json +{ + "caldav": { + "url": "https:///remote.php/dav/calendars//", + "username": "", + "app_password": "...", + "inbox_list": "Inbox", + "someday_list": "Someday", + "exclude_lists": [ + "..." + ] + } +} +``` + +A complete example: ```json { @@ -106,21 +228,16 @@ unreadable configuration must never pass as "no tracker". "superproductivity": { "access": "api", "api_base_url": "http://127.0.0.1:3876", - "api_token": "" + "api_token": "..." } } ``` + -`provider` picks the adapter. The stack ships `superproductivity` and `caldav`. - -The `thresholds` block holds the review's three thresholds (configuration, not schema): when a -waiting-for counts as overdue, how old a tracker project without a `kb/` page has to be before it is -reported, and when a someday item counts as stale. A value the provider cannot supply at all - a +The `thresholds` block is configuration, not schema. A value the provider cannot supply at all - a waiting item with no follow-up date, a tracker project with no creation date - is not skipped silently but reported as its own finding (`waiting_no_follow_up`/`project_age_unknown`). -The provider block of the same name holds the connection details. - ### Super Productivity `access` is **mandatory and has no fallback**. It chooses between two mutually exclusive ways in: @@ -129,20 +246,10 @@ The provider block of the same name holds the connection details. under `backups_dir`. That works without the app running, but only reads. The tracker cannot be written from there. - ```json - "superproductivity": { - "access": "snapshot", - "backups_dir": "~/.config/superProductivity/backups" - } - ``` - - **`access: "api"`** for a desktop instance: it talks only to the local REST API. That answers only while the app runs, but it gives the current state and carries the write path. `api_token` is required, because every endpoint except `GET /health` demands `Authorization: Bearer `. -The block names only the fields of its own way in. A `backups_dir` beside `access: "api"`, or an -`api_token` beside `access: "snapshot"`, is refused when the configuration is read, not ignored. - What can write: - `tools/wikitool new project` creates a tracker project of the same name only with `access: "api"`, @@ -159,27 +266,13 @@ from there. `tools/wikitool task list --project` only reads, so it answers with `caldav` is the standards-based second adapter (RFC 4791/5545). It is verified against Nextcloud Tasks, with iOS *Reminders* as the mobile client, and built for that split: tick items off on the phone, maintain them at the desk. Unlike Super Productivity, there is only one way in. CalDAV is -always a network access, so no `access` field is needed: +always a network access, so no `access` field is needed. -```json - "caldav": { - "url": "https:///remote.php/dav/calendars//", - "username": "", - "app_password": "", - "inbox_list": "Inbox", - "someday_list": "Someday", - "exclude_lists": [""] - } -``` - -- `url` points at the account's CalDAV calendar home set. It may be an alias - Nextcloud accepts a - short name there - because every later address comes only from the `href`s the server returns, - never from this URL joined with a name. -- `username` is the login name, which can differ from the user id in the URL. A Nextcloud app - password is recommended, not the account password. +- `url` may be an alias - Nextcloud accepts a short name there - because every later address comes + only from the `href`s the server returns, never from this URL joined with a name. - `inbox_list` and `someday_list` name the two fixed lists, exactly one of each per instance. -- `exclude_lists` leaves out existing plain task lists that are not projects. A list with a - `VEVENT` component never counts as a project anyway. +- A list with a `VEVENT` component never counts as a project, so `exclude_lists` is needed only for + plain task lists. A project is a list whose only supported component is `VTODO`. `tools/wikitool new project` creates it automatically with `MKCALENDAR` - without asking, unlike Super Productivity, because CalDAV has a @@ -199,5 +292,41 @@ a list. Remove them by hand in Nextcloud when you want to. ## MCP submissions `.wikitool-upload.json` in the served corpus turns on the MCP read server's optional `submit` tool. -Its shape, and what the authentication in front of the server has to guarantee before you create -it, are in [Set up the MCP read server](../how-to/set-up-the-mcp-server.md#optional-the-submit-path). +What the authentication in front of the server has to guarantee before you create it is in +[Set up the MCP read server](../how-to/set-up-the-mcp-server.md#optional-the-submit-path). + + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `schema` | any | no | - | Accepted and not read. Conventionally `1`; no version of this file is told apart by it yet. | +| `identity_header` | string | no | `"X-Forwarded-User"` | The request header the authenticating middleware sets to the submitter's identity on every request. An empty value means the default. | +| `max_bytes` | positive integer | yes | - | Largest single submission, in bytes. A number written as a string is accepted. | +| `allowed_extensions` | array of strings | yes | - | The file extensions `submit` accepts, each starting with `.`. Compared case-insensitively. Entries are read as strings and lower-cased; duplicates collapse. | +| `quota` | object | yes | - | Per-submitter limits over any rolling 24 hours. | +| `quota.submissions_per_day` | positive integer | yes | - | Submissions one submitter may make. A number written as a string is accepted. | +| `quota.bytes_per_day` | positive integer | yes | - | Bytes one submitter may submit. A number written as a string is accepted. | + +- **Absent:** The `submit` tool is not registered: the write path does not exist. +- **Malformed:** An error. The MCP server does not start and `doctor` reports `FAIL`. A broken file is never read as "no limits". +- **Unknown keys:** Ignored. + +Example: + +```json +{ + "schema": 1, + "identity_header": "X-Forwarded-User", + "max_bytes": 10485760, + "allowed_extensions": [ + ".md", + ".pdf" + ], + "quota": { + "submissions_per_day": 20, + "bytes_per_day": 52428800 + } +} +``` + + +Size and quota are your deployment's decision, not defaults worth copying blindly. diff --git a/instructions/dev/doc-pull-through.md b/instructions/dev/doc-pull-through.md index 6a2694d..8fd8b0c 100644 --- a/instructions/dev/doc-pull-through.md +++ b/instructions/dev/doc-pull-through.md @@ -40,7 +40,7 @@ touched; a row that does not apply needs no action. | The reasoning behind a gate, boundary, or design decision | The `docs/explanation/` page that carries it, if one exists (AGENTS.md § File naming lists all six reached from AGENTS.md itself, plus a seventh reached only from CLAUDE.md). **A decision with no page yet is the gap worth closing**: reasoning that lives only in a Gitea issue never ships - `dist export` carries `docs/explanation/` and no issue tracker, so a distributed instance gets the mechanism without the why | | A task-tracker adapter (`tools/chemenu/tasks/`), its recorded fixtures, or the live suite | [instructions/dev/tracker-testing.md](tracker-testing.md), and `MANIFEST.json` beside the fixtures when they were re-recorded | | A skill's own step sequence or catalogue | The skill's `SKILL.md` source under `instructions//` or `instructions/dev//`; for a shipped skill (not under `instructions/dev/`) whose name or `description` changed, or one added or removed, `wikitool docs reference --apply` for the skill catalogue `docs/reference/skills.md` | - | A per-checkout configuration file an instance owns (`.wikitool-tasks.json`, `.wikitool-telemetry.json`, `.wikitool-remotes.json`, `.wikitool-upload.json`) | [docs/reference/configuration.md](../../docs/reference/configuration.md), where an operator looks the shape up - written by hand, since nothing generates it; the [setup-instance.md](../setup-instance.md) decision point that offers it during setup; and `doctor`'s own row in [tools/CONTRACT.md](../../tools/CONTRACT.md), since `doctor` is what reports the file's state | + | A per-checkout configuration file an instance owns (`.wikitool-tasks.json`, `.wikitool-telemetry.json`, `.wikitool-remotes.json`, `.wikitool-upload.json`) | Its shape - a field, a default, what is required or tolerated - is its record in `tools/chemenu/config_shapes.py`, which the loader reads; change it there, then `wikitool docs reference --apply` for its region in [docs/reference/configuration.md](../../docs/reference/configuration.md). What a setting *does* is the hand-written prose around that region. Also the [setup-instance.md](../setup-instance.md) decision point that offers the file during setup, and `doctor`'s own row in [tools/CONTRACT.md](../../tools/CONTRACT.md), since `doctor` is what reports the file's state | | An installation instruction - [preflight.md](../preflight.md), [setup-instance.md](../setup-instance.md), [bootstrap.md](../bootstrap.md), [upgrade-instance.md](../upgrade-instance.md) - or `tools/prerequisites.txt` | [INSTALL.md](../../INSTALL.md), the human guide to the same procedure - for `upgrade-instance.md`, [docs/how-to/update-an-instance.md](../../docs/how-to/update-an-instance.md) instead. Read it against the instruction: what to prepare, the sentence for the agent, what the agent asks, where it stops and why. `docs verify` checks only the two enumerable overlaps - the prerequisites lists, which `wikitool docs prerequisites --apply` regenerates from the manifest, and the setup questions: a question the agent asks the user carries `` where it is asked in `setup-instance.md`, and `INSTALL.md` § "What the agent asks you" names it with the same marker. Every other sentence is this session's to compare. `INSTALL.md` does not retell the steps, so a change to their order or wording alone moves nothing there | | [dev-setup.md](dev-setup.md), the release workflow (`version bump`/`version release`/`publish`/CI), or another developer-facing process | [docs/dev/development.md](../../docs/dev/development.md), read against it the same way - nothing checks this pair at all. A page under `docs/dev/` is never shipped, so it may link into `instructions/dev/` and cite issue numbers | | A new page type the stack requires, or a new collection | Its type-spec and `COLLECTION.md` (both as the `.template` an instance adopts), the collection table in [kb/CONTRACT.md](../../kb/CONTRACT.md), and **both adoption paths**: [setup-instance.md](../setup-instance.md) for a fresh instance and [upgrade-instance.md](../upgrade-instance.md) for an existing one, where an unadopted template is what `docs verify` refuses. A new or changed `root: kb` type-spec or its schema - description, `layout:`, subtype enum, required fields - also needs `wikitool docs reference --apply` for `docs/reference/page-types.md` | diff --git a/instructions/setup-instance.md b/instructions/setup-instance.md index ea13826..f11998c 100644 --- a/instructions/setup-instance.md +++ b/instructions/setup-instance.md @@ -281,8 +281,9 @@ one it needs before that. { "enabled": true } ``` - `max_session_bytes` (default 5 MiB) and `keep_sessions` (default 250) are optional in the - same file; most instances need not touch them. If no, do nothing - the default is already + The file's two optional caps and their defaults are in + [docs/reference/configuration.md](../docs/reference/configuration.md#telemetry); most instances + need not touch them. If no, do nothing - the default is already off, and no file is created. `WIKI_TRACE` still overrides in both directions, should a single session need to differ. diff --git a/tools/CONTRACT.md b/tools/CONTRACT.md index edc567c..e3b2aef 100644 --- a/tools/CONTRACT.md +++ b/tools/CONTRACT.md @@ -2559,7 +2559,7 @@ Check the docs that mirror the code. - Every reference file `docs toc` covers carries the current table-of-contents region for its own headings - missing and stale are one check. - Every relative markdown link in one of those reference files resolves to an existing file. A target's `#anchor` suffix is stripped first, and code fences and inline code spans are masked before scanning, so link syntax shown as an example is not mistaken for a real reference. - Every file under `docs/` sits in one of its areas - `tutorials/`, `how-to/`, `reference/`, `explanation/` or `dev/` - never directly in `docs/` or in another directory. -- If `docs/` exists: every generated region of the human reference under `docs/reference/` matches what `docs reference` would write, every `cli_contract.GROUPS` entry has its page under `docs/reference/commands/`, and no page there belongs to no group. The skill catalogue and the page types are checked only in the origin repo (no `.wikitool-release.json`): they describe the stack's shipped defaults, which an instance's own type-specs and instructions may differ from. +- If `docs/` exists: every generated region of the human reference under `docs/reference/` matches what `docs reference` would write, every `cli_contract.GROUPS` entry has its page under `docs/reference/commands/`, and no page there belongs to no group. That includes the four configuration regions in `docs/reference/configuration.md`. The skill catalogue and the page types are checked only in the origin repo (no `.wikitool-release.json`): they describe the stack's shipped defaults, which an instance's own type-specs and instructions may differ from. - `INSTALL.md` carries one generated `` region per platform value of `tools/prerequisites.txt` (`prerequisites-` for a platform-specific one), each current; and the `` markers in `instructions/setup-instance.md` and `INSTALL.md` name the same set of keys, so a question the agent asks is never one the human guide leaves out, nor the reverse. - `VERSION` parses and the newest versioned `CHANGES.md` entry names it. An entry that crosses the compatibility boundary from the last release carries `**Breaking Change:**`, has a migration document or `**Migration:** none required`, and its `**Migration:**` line names every required migration document targeting it. - The newest versioned `CHANGES.md` entry is at most 32 000 bytes (UTF-8) as `version notes` prints it, and each of its `###` sections is exactly one paragraph of at most 1 000 bytes, heading line excluded. Older entries and a changelog with no versioned entry are not checked. @@ -2744,6 +2744,7 @@ Regenerate the human reference under `docs/reference/`. - Rewrites the generated region of every page of the human command reference from the `cli_contract` records: one `docs/reference/commands/.md` per `cli_contract.GROUPS` entry (``), the overview in `docs/reference/commands/README.md` (``), and the gates and exit-42 causes in `docs/reference/exit-codes.md` (``). - Also, in the origin repo only, the skill catalogue `docs/reference/skills.md` (``: one row per shipped `instructions//SKILL.md`, its description split at the sentence starting "Use when") and the page types `docs/reference/page-types.md` (``: one section per `root: kb` type-spec, from what `types describe` reads). Both describe the stack's shipped defaults. In an instance - a tree with `.wikitool-release.json` - they are left alone and the output says so: an instance's adopted type-specs and own instructions may differ, and rewriting a shipped page there would block the next `dist upgrade`. +- Also the shapes of the four hand-written configuration files in `docs/reference/configuration.md`, one region each (``, `-remotes`, `-tasks`, `-upload`) inside the hand-written section on that file, rendered from the records in `chemenu.config_shapes` the loaders read the files with. Those records ship with `tools/`, so these regions are written and checked in an instance as well. Several regions in one page are refreshed one after the other into the same text. - Renders each command's summary, synopsis, options, examples, notes, properties in words, exit codes with their causes and see-also - but not the ON FAILURE reactions or NEVER, which address an agent; `tools/CONTRACT.md` keeps those. - Refreshes the table of contents of each page it writes in the same step. - Never creates a page or places a region: a page's title and one-sentence head are written by hand. A group's page is named after the group - lower case, every run of other characters one hyphen. @@ -3636,7 +3637,7 @@ Check that this instance is correctly configured. - For a configured `superproductivity` provider, the configured `access` path's own state: `access: "api"` reports whether its local REST API answers `GET /health` with a ready renderer right now, `access: "snapshot"` whether a backup file is ready. The other access path is never attempted, and neither state is ever a `FAIL`. - For a configured `caldav` provider, whether the server is reachable and Basic auth succeeds - never a `FAIL`; only a broken config block is. - Session id source: `OK` for `WIKITOOL_SESSION_ID` or a registered harness variable, `WARN` only for the bare parent-pid fallback. -- Telemetry: on or off and why - installation-form default, `.wikitool-telemetry.json`, or `WIKI_TRACE` - and the current session's count and byte total against both caps; never a `FAIL`. +- Telemetry: on or off and why - installation-form default, `.wikitool-telemetry.json`, or `WIKI_TRACE` - and the current session's count and byte total against both caps; never a `FAIL`. A `WARN` when `.wikitool-telemetry.json` is not read as written - unreadable JSON, not an object, `enabled` not a JSON boolean, a cap not a positive integer, or an unknown key - naming the field; telemetry itself drops or reinterprets such a value without a word, so this is where it shows. - Exits 1 only on a `FAIL`; a missing remote, session id or `VERSION` is a `WARN`, not a fault. - Read-only and exempt from the Iteration Budget Gate. diff --git a/tools/README.md b/tools/README.md index eedeeb8..893f35f 100644 --- a/tools/README.md +++ b/tools/README.md @@ -83,13 +83,14 @@ tools/ cli.py Typer app: registers every command, runs the budget gate, renders `-h`/`--help` from cli_contract cli_contract.py one data record per command (name, synopsis, properties, exit status) - the source `-h`, the index, CONTRACT.md's generated region and the human reference under docs/reference/ render from config.py repo layout: root resolution and every path under it + config_shapes.py one record per hand-written .wikitool-*.json (telemetry, remotes, upload, tasks): what its loader accepts, its defaults and `Expected:` text, and what docs/reference/configuration.md renders; stdlib-only, since the telemetry policy reads it api.py the in-process entry point - point Chemenu at a corpus and read it errors.py ChemenuError / ValidationError / BackendError toolpaths.py where git and rg are started from: .wikitool-tools.json, bare name only without the file filelock.py an exclusive lock on an open file, flock on POSIX and msvcrt on Windows - the only module that imports either prerequisites.py prerequisites.txt read from Python, plus the platform and long-path questions `doctor` asks install_doc.py INSTALL.md held to the instructions: its prerequisites lists generated from prerequisites.txt, its setup questions matched to setup-instance.md's markers - reference_doc.py the human reference under docs/reference/: one page per command group, an overview and the exit-42 causes, rendered from the records without what addresses an agent; and, in the origin repo only, the skill catalogue and the page types, rendered from the shipped SKILL.md files and type-specs + reference_doc.py the human reference under docs/reference/: one page per command group, an overview and the exit-42 causes, rendered from the records without what addresses an agent; and, in the origin repo only, the skill catalogue and the page types, rendered from the shipped SKILL.md files and type-specs; and the configuration-file shapes, from config_shapes.py corpus_cache.py one parsed corpus per commit, never cached while the tree is dirty kb_scan.py page iteration/loading over kb/ blocks.py generated regions in a page body, found by marker rather than by heading diff --git a/tools/chemenu/commands/docs_verify.py b/tools/chemenu/commands/docs_verify.py index 549617e..8e75384 100644 --- a/tools/chemenu/commands/docs_verify.py +++ b/tools/chemenu/commands/docs_verify.py @@ -1296,7 +1296,8 @@ def check_breaking_change_for_boundary() -> list[str]: "If `docs/` exists: every generated region of the human reference under " "`docs/reference/` matches what `docs reference` would write, every `cli_contract.GROUPS` " "entry has its page under `docs/reference/commands/`, and no page there belongs to no " - "group. The skill catalogue and the page types are checked only in the origin repo " + "group. That includes the four configuration regions in " + "`docs/reference/configuration.md`. The skill catalogue and the page types are checked only in the origin repo " "(no `.wikitool-release.json`): they describe the stack's shipped defaults, which an " "instance's own type-specs and instructions may differ from.", "`INSTALL.md` carries one generated `` region per " @@ -1704,6 +1705,13 @@ def contract_command( "shipped defaults. In an instance - a tree with `.wikitool-release.json` - they are left " "alone and the output says so: an instance's adopted type-specs and own instructions may " "differ, and rewriting a shipped page there would block the next `dist upgrade`.", + "Also the shapes of the four hand-written configuration files in " + "`docs/reference/configuration.md`, one region each " + "(``, `-remotes`, `-tasks`, `-upload`) inside " + "the hand-written section on that file, rendered from the records in " + "`chemenu.config_shapes` the loaders read the files with. Those records ship with " + "`tools/`, so these regions are written and checked in an instance as well. Several " + "regions in one page are refreshed one after the other into the same text.", "Renders each command's summary, synopsis, options, examples, notes, properties in words, " "exit codes with their causes and see-also - but not the ON FAILURE reactions or NEVER, " "which address an agent; `tools/CONTRACT.md` keeps those.", @@ -1737,7 +1745,6 @@ def reference_command( ): """Regenerate the human reference under `docs/reference/`.""" missing = [] - changed = [] generated, left_alone = reference_doc.applicable(reference_doc.regions()) if left_alone: typer.echo( @@ -1746,17 +1753,22 @@ def reference_command( + " - they describe the stack's shipped defaults; `wikitool types describe ` " "and `wikitool instructions list` show this instance's own." ) + # A page may carry several regions (`configuration.md` has one per file), + # so each is refreshed into the text the previous one left, not into what + # is on disk - or only the last region of a page would survive the write. + texts: dict[Path, tuple[str, str]] = {} for region in generated: path = config.ROOT / region.path if not path.is_file(): missing.append(f"{region.path} (create it with `{blocks.open_marker(region.name)}`)") continue - before = path.read_text(encoding="utf-8") - after = reference_doc.refresh(before, region) + before, current = texts.get(path) or (path.read_text(encoding="utf-8"),) * 2 + after = reference_doc.refresh(current, region) if after is None: missing.append(f"{region.path} (add `{blocks.open_marker(region.name)}`)") - elif after != before: - changed.append((path, after)) + else: + texts[path] = (before, after) + changed = [(path, after) for path, (before, after) in texts.items() if after != before] if missing: fail( diff --git a/tools/chemenu/commands/doctor.py b/tools/chemenu/commands/doctor.py index 2ee3020..f127920 100644 --- a/tools/chemenu/commands/doctor.py +++ b/tools/chemenu/commands/doctor.py @@ -21,7 +21,7 @@ from typing import Optional import typer from rich.console import Console -from chemenu import cli_contract, config, conventions, kb_collections, prerequisites, toolpaths, version as version_mod +from chemenu import cli_contract, config, config_shapes, conventions, kb_collections, prerequisites, toolpaths, version as version_mod from chemenu.commands import git_publish, instructions_cmd from chemenu.commands._util import rel_path from chemenu.session import ENV_VAR as SESSION_ENV_VAR @@ -466,7 +466,8 @@ def check_telemetry() -> Check: Never `FAIL`s, the same line `check_publish_remotes` and `check_environment` draw: both an enabled and a disabled tree are legitimate states, and a `FAIL` would make one of them mandatory by the - back door. The tree walk below is fine here - `doctor` is not a hot path, + back door. It `WARN`s on a `.wikitool-telemetry.json` the policy does not + read as written, since the policy itself drops that in silence. The tree walk below is fine here - `doctor` is not a hot path, unlike the `stat` `emit()` does on every append. """ from chemenu.telemetry import policy as telemetry_policy @@ -482,12 +483,38 @@ def check_telemetry() -> Check: else 0 ) state = "on" if pol.enabled else "off" - return Check( - "telemetry", "OK", + detail = ( f"{state} ({pol.reason}); {session_count}/{pol.keep_sessions} session(s), " f"{total_bytes:,} byte(s) under {rel_path(root)} " - f"(cap {pol.max_session_bytes:,} byte(s)/session)", + f"(cap {pol.max_session_bytes:,} byte(s)/session)" ) + deviations = _telemetry_file_deviations() + if deviations: + return Check( + "telemetry", "WARN", + f"{detail}. {config.TELEMETRY_FILENAME} is not read as written: {'; '.join(deviations)}", + f"Correct {config.TELEMETRY_FILENAME} to the shape in " + "docs/reference/configuration.md#telemetry, or delete it to get the default", + ) + return Check("telemetry", "OK", detail) + + +def _telemetry_file_deviations() -> list[str]: + """What the telemetry policy silently drops or reinterprets in this + checkout's `.wikitool-telemetry.json` - nothing when the file is absent. + + The policy has to stay silent (a hook handler reads it on every tool + call); this is where the silence becomes visible.""" + path = config.ROOT / config.TELEMETRY_FILENAME + if not path.is_file(): + return [] + try: + data = _json.loads(path.read_text(encoding="utf-8")) + except (OSError, _json.JSONDecodeError) as exc: + return [f"unreadable ({exc}), so the whole file is ignored"] + if not isinstance(data, dict): + return ["not a JSON object, so the whole file is ignored"] + return config_shapes.deviations(config_shapes.TELEMETRY, data) def check_upload_intake() -> Check: @@ -788,7 +815,10 @@ def run_doctor() -> list[Check]: "`WARN` only for the bare parent-pid fallback.", "Telemetry: on or off and why - installation-form default, `.wikitool-telemetry.json`, " "or `WIKI_TRACE` - and the current session's count and byte total against both caps; " - "never a `FAIL`.", + "never a `FAIL`. A `WARN` when `.wikitool-telemetry.json` is not read as written - " + "unreadable JSON, not an object, `enabled` not a JSON boolean, a cap not a positive " + "integer, or an unknown key - naming the field; telemetry itself drops or reinterprets " + "such a value without a word, so this is where it shows.", "Exits 1 only on a `FAIL`; a missing remote, session id or `VERSION` is a `WARN`, not a " "fault.", "Read-only and exempt from the Iteration Budget Gate.", diff --git a/tools/chemenu/commands/git_publish.py b/tools/chemenu/commands/git_publish.py index b2114e1..62afc11 100644 --- a/tools/chemenu/commands/git_publish.py +++ b/tools/chemenu/commands/git_publish.py @@ -62,7 +62,7 @@ from typing import NamedTuple, Optional import typer -from chemenu import cli_contract, config, toolpaths +from chemenu import cli_contract, config, config_shapes, toolpaths from chemenu.commands._util import fail, needs_clearance, success from chemenu.telemetry import emit @@ -123,14 +123,15 @@ def read_allowed_push_urls() -> Optional[list[str]]: "fix the file or delete it deliberately." ) return None - urls = data.get("allowed_push_urls") - if not isinstance(urls, list) or not all(isinstance(u, str) for u in urls): + shape = config_shapes.REMOTES + checked = config_shapes.check(shape, data) + if checked.problems: fail( - f"{config.PUBLISH_REMOTES_FILENAME} has no usable `allowed_push_urls` list of " - "strings. Expected: {\"schema\": 1, \"allowed_push_urls\": [\"\"]}" + f"{config.PUBLISH_REMOTES_FILENAME} has no usable allowed-URL list: {checked.message()}. " + f"Expected: {config_shapes.render_expected(shape)}" ) return None - return urls + return checked.values["allowed_push_urls"] def push_url_for(remote: str) -> Optional[str]: diff --git a/tools/chemenu/config.py b/tools/chemenu/config.py index 43404f6..84f6c4f 100644 --- a/tools/chemenu/config.py +++ b/tools/chemenu/config.py @@ -222,15 +222,17 @@ LICENSE_FILES = ("LICENSE", "LICENSE-CONTENT", "NOTICE") RELEASE_STAMP_FILENAME = ".wikitool-release.json" # Per-checkout telemetry opt-in/opt-out plus its two quantity caps (see -# `telemetry/policy.py`). Same shape as `PUBLISH_REMOTES_FILENAME` below: it +# `telemetry/policy.py`). Same posture as `PUBLISH_REMOTES_FILENAME` below: it # answers a question about *this* checkout, so it is per-checkout and # gitignored, ships no `.template`, and its absence is a legitimate state - # the installation-form default (keyed off `RELEASE_STAMP_FILENAME`) applies. # `instructions/setup-instance.md`'s Telemetry decision point writes it from -# the operator's answer. +# the operator's answer. Its fields are the record `config_shapes.TELEMETRY`, +# as for the three files below - each has exactly one. TELEMETRY_FILENAME = ".wikitool-telemetry.json" -# Which push targets `publish` may write to, for a checkout that says so. The +# Which push targets `publish` may write to, for a checkout that says so +# (fields: `config_shapes.REMOTES`). The # danger this addresses is one checkout's content reaching another checkout's # remote - a private instance pushing its own `kb/` to a public repository, # where it cannot be taken back. @@ -246,8 +248,8 @@ TELEMETRY_FILENAME = ".wikitool-telemetry.json" # instructions/gates.md. PUBLISH_REMOTES_FILENAME = ".wikitool-remotes.json" -# Opt-in for the MCP server's `submit` tool (Gitea #32): identity header name, -# size deckel, extension allowlist, per-submitter quota. Same shape as the two +# Opt-in for the MCP server's `submit` tool (Gitea #32); its fields are the +# record `config_shapes.UPLOAD`. Same posture as the two # above - per-checkout, gitignored, no `.template` - but its absence means # something stronger than "unrestricted": **the write path does not exist at # all**, the tool is not registered. The safe direction, and a structural @@ -256,7 +258,8 @@ UPLOAD_CONFIG_FILENAME = ".wikitool-upload.json" # The task-tracker provider opt-in (Gitea #124, D25/D30): which provider this # instance's GTD review reads/writes through, its connection details, and the -# review's three staleness thresholds. Same shape as the three files above - +# review's three staleness thresholds - fields in `config_shapes.TASKS`. Same +# posture as the three files above - # per-checkout, gitignored once a credential lands in it, no `.template` - and # its absence is a legitimate state, the same posture `UPLOAD_CONFIG_FILENAME` # takes: an instance with no tracker configured runs `doctor` and everything diff --git a/tools/chemenu/config_shapes.py b/tools/chemenu/config_shapes.py new file mode 100644 index 0000000..d8b1b14 --- /dev/null +++ b/tools/chemenu/config_shapes.py @@ -0,0 +1,596 @@ +"""The shapes of the per-checkout configuration files an operator writes by hand. + +Four files - `.wikitool-telemetry.json`, `.wikitool-remotes.json`, +`.wikitool-upload.json` and `.wikitool-tasks.json` (names in `chemenu.config`) +- each have exactly one record here, and that record is the only place their +shape is written down. The loader that reads a file takes its field names, +required fields, defaults, coercions, conditions and the `Expected:` text of its +error messages from the record; `docs reference --apply` renders the same record +into `docs/reference/configuration.md`. A second description of a shape, in a +loader or in prose, is what used to drift. + +The records describe what the loaders **accept today**, leniency included: +`int()` on a number given as a string, `bool()` on `enabled`, unknown top-level +keys ignored. Tightening any of that would reject a file an instance already +has, which is a different kind of change than writing the shape down. + +`check()` never raises. Each loader keeps its own posture towards a problem - +`publish` stops through `fail()`, the upload and tracker loaders raise +`ValidationError`, and telemetry drops what it cannot use in silence, because a +hook handler reads it on every tool call. `deviations()` is the strict reading +`doctor` uses to report what that silence would otherwise hide. + +The three machine-written files beside them - `.wikitool-release.json`, +`.wikitool-kb.json`, `.wikitool-tools.json` - are deliberately not here: nobody +writes them by hand, so they have no shape for an operator to look up. + +Stdlib only, and importing nothing from this package but `chemenu.config`: +`chemenu.telemetry.policy` reads its record, and a hook handler imports that on +every tool call without the venv. +""" +from __future__ import annotations + +import json +from dataclasses import dataclass, field as dc_field +from typing import Any, Callable, Optional + +from chemenu import config + +IGNORE = "ignore" +REJECT = "reject" + +_ABSENT = object() + + +# --------------------------------------------------------------------------- +# The record types +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Field: + """One key of a configuration object. + + `json_type` is the JSON type a conforming file uses (`boolean`, `integer`, + `string`, `array`, `object`, or None for any). `coerce` is what the loader + applies to the value it finds - it may accept more than `json_type`, which + is the leniency `deviations()` reports - and raises `TypeError`/`ValueError` + on a value it cannot use. `check` then judges the coerced value and returns + a problem, or None. A field with `fields` is a nested object.""" + + name: str + type: str + description: str + required: bool = False + default: Any = None + default_text: str = "" + json_type: Optional[str] = None + coerce: Optional[Callable[[Any], Any]] = None + check: Optional[Callable[[Any], Optional[str]]] = None + choices: tuple[str, ...] = () + empty_is_missing: bool = False + required_why: str = "" + lenient: str = "" + fields: tuple["Field", ...] = () + + +@dataclass(frozen=True) +class Shape: + """One configuration object: a whole file, or a provider's section of one. + + `key` is the file name for a file and the section's key for a section. + `discriminator` names a field whose value picks one of `variants` - an + extra set of fields valid only alongside that value. `section_key` names a + field whose value picks one of `sections`, a nested object stored under the + chosen value's own key.""" + + key: str + title: str + purpose: str + fields: tuple[Field, ...] + examples: tuple[dict, ...] + absent: str = "" + malformed: str = "" + unknown_keys: str = IGNORE + unknown_text: str = "" + unknown_why: str = "" + discriminator: str = "" + variants: tuple[tuple[str, tuple[Field, ...]], ...] = () + section_key: str = "" + sections: tuple["Shape", ...] = () + + def field(self, name: str) -> Field: + for candidate in self.fields + tuple(f for _, fs in self.variants for f in fs): + if candidate.name == name: + return candidate + raise KeyError(name) + + def variant_fields(self, value: Any) -> Optional[tuple[Field, ...]]: + return dict(self.variants).get(value) + + def choices(self, field: Field) -> tuple[str, ...]: + """The values `field` may take: its own list, or - for the + discriminator and the section key - the variants and sections that + exist, so a new one cannot be added without the choice following.""" + if field.name == self.discriminator: + return tuple(value for value, _ in self.variants) + if field.name == self.section_key: + return tuple(s.key for s in self.sections) + return field.choices + + +@dataclass(frozen=True) +class Problem: + field: str + message: str + + +@dataclass +class Checked: + """What `check()` found: the coerced values (defaults filled in), the + fields the file actually set and the loader accepted, the problems a + loader that rejects would reject on, the fields accepted only by coercion, + and the keys no field names.""" + + values: dict = dc_field(default_factory=dict) + given: set = dc_field(default_factory=set) + problems: list = dc_field(default_factory=list) + coerced: list = dc_field(default_factory=list) + unknown: list = dc_field(default_factory=list) + + def message(self) -> str: + return "; ".join(p.message for p in self.problems) + + +# --------------------------------------------------------------------------- +# Coercions and conditions +# --------------------------------------------------------------------------- + + +def _exact_str(value: Any) -> str: + if not isinstance(value, str): + raise TypeError("not a string") + return value + + +def _str_list(value: Any) -> list[str]: + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise TypeError("not a list of strings") + return list(value) + + +def _extensions(value: Any) -> tuple[str, ...]: + return tuple(sorted({str(ext).lower() for ext in value})) + + +def _positive(value: int) -> Optional[str]: + return None if value > 0 else "must be positive" + + +def _dotted_extensions(value: tuple[str, ...]) -> Optional[str]: + if not value: + return "must not be empty" + not_dotted = [ext for ext in value if not ext.startswith(".")] + return f"must each start with '.': {not_dotted}" if not_dotted else None + + +def _positive_integer(name: str, description: str, **kwargs: Any) -> Field: + return Field( + name, "positive integer", description, json_type="integer", coerce=int, + check=_positive, lenient="A number written as a string is accepted.", **kwargs, + ) + + +_SCHEMA = Field( + "schema", "any", "Accepted and not read. Conventionally `1`; no version of this file is " + "told apart by it yet.", +) + + +# --------------------------------------------------------------------------- +# The records +# --------------------------------------------------------------------------- + +TELEMETRY = Shape( + key=config.TELEMETRY_FILENAME, + title="Telemetry", + purpose="Turns telemetry on or off for this checkout, against the installation-form default, " + "and sets its two quantity caps.", + fields=( + _SCHEMA, + Field( + "enabled", "boolean", "Telemetry on or off. `WIKI_TRACE` beats it in both directions.", + default_text="the installation-form default", json_type="boolean", coerce=bool, + lenient="Any other value is read by its truthiness, so `\"no\"` means on.", + ), + _positive_integer( + "max_session_bytes", "Byte cap per session trace. `WIKI_TRACE_MAX_SESSION_BYTES` beats it.", + default=5 * 1024 * 1024, + ), + _positive_integer( + "keep_sessions", "How many session directories retention keeps. " + "`WIKI_TRACE_KEEP_SESSIONS` beats it.", default=250, + ), + ), + examples=({"enabled": True, "max_session_bytes": 5242880, "keep_sessions": 250},), + absent="The installation-form default applies: on in a clone of the origin repository, off in " + "an instance.", + malformed="Never an error. Unreadable JSON or a non-object counts as an absent file, and a cap " + "that is not a positive integer as an absent cap. `doctor` reports each such deviation as a " + "`WARN` under `telemetry`, naming the field.", + unknown_text="Ignored; `doctor` reports them as a `WARN`.", +) + +REMOTES = Shape( + key=config.PUBLISH_REMOTES_FILENAME, + title="Allowed push URLs", + purpose="Restricts the URLs `publish` may push to.", + fields=( + _SCHEMA, + Field( + "allowed_push_urls", "array of strings", "The push URLs `publish` may write to, " + "compared with the URL the remote resolves to for a push - not with its name. An " + "empty list allows no push at all.", + required=True, json_type="array", coerce=_str_list, + ), + ), + examples=({"schema": 1, "allowed_push_urls": [""]},), + absent="Any remote is allowed; the Publish-Remote Gate does not apply.", + malformed="`publish` stops with an error before committing anything. A broken file is never " + "read as \"no restriction\".", + unknown_text="Ignored.", +) + +UPLOAD = Shape( + key=config.UPLOAD_CONFIG_FILENAME, + title="MCP submissions", + purpose="Turns on the MCP read server's `submit` tool and sets its limits.", + fields=( + _SCHEMA, + Field( + "identity_header", "string", "The request header the authenticating middleware sets to " + "the submitter's identity on every request.", + default="X-Forwarded-User", json_type="string", coerce=str, empty_is_missing=True, + lenient="An empty value means the default.", + ), + _positive_integer("max_bytes", "Largest single submission, in bytes.", required=True), + Field( + "allowed_extensions", "array of strings", "The file extensions `submit` accepts, each " + "starting with `.`. Compared case-insensitively.", + required=True, json_type="array", coerce=_extensions, check=_dotted_extensions, + lenient="Entries are read as strings and lower-cased; duplicates collapse.", + ), + Field( + "quota", "object", "Per-submitter limits over any rolling 24 hours.", + required=True, json_type="object", + fields=( + _positive_integer("submissions_per_day", "Submissions one submitter may make.", + required=True), + _positive_integer("bytes_per_day", "Bytes one submitter may submit.", required=True), + ), + ), + ), + examples=({ + "schema": 1, "identity_header": "X-Forwarded-User", "max_bytes": 10485760, + "allowed_extensions": [".md", ".pdf"], + "quota": {"submissions_per_day": 20, "bytes_per_day": 52428800}, + },), + absent="The `submit` tool is not registered: the write path does not exist.", + malformed="An error. The MCP server does not start and `doctor` reports `FAIL`. A broken file " + "is never read as \"no limits\".", + unknown_text="Ignored.", +) + +SUPERPRODUCTIVITY = Shape( + key="superproductivity", + title="Super Productivity", + purpose="Connection settings for the Super Productivity adapter.", + fields=( + Field( + "access", "string", "Which way in: `api`, the local REST API (reads the current state " + "and carries the write path), or `snapshot`, the newest backup snapshot (read-only).", + required=True, json_type="string", + required_why="there is no default and no fallback between the two", + ), + ), + discriminator="access", + variants=( + ("api", ( + Field( + "api_base_url", "string", "Where the app's REST API listens.", + default="http://127.0.0.1:3876", json_type="string", coerce=str, + empty_is_missing=True, lenient="An empty value means the default.", + ), + Field( + "api_token", "string", "The token from the app's settings.", + required=True, json_type="string", coerce=_exact_str, empty_is_missing=True, + required_why="every endpoint but GET /health requires Authorization: Bearer ", + ), + )), + ("snapshot", ( + Field( + "backups_dir", "string", "The directory the app writes its backup snapshots to; " + "`~` is expanded.", + required=True, json_type="string", coerce=_exact_str, empty_is_missing=True, + required_why="the read path has nothing to read otherwise", + ), + )), + ), + examples=( + {"access": "api", "api_base_url": "http://127.0.0.1:3876", "api_token": "..."}, + {"access": "snapshot", "backups_dir": "~/.config/superProductivity/backups"}, + ), + unknown_keys=REJECT, + unknown_why="a section names only one access path's own fields", + unknown_text="Rejected: a section names only one access path's own fields, so a `backups_dir` " + "beside `access: \"api\"` is an error, not ignored.", +) + +CALDAV = Shape( + key="caldav", + title="CalDAV", + purpose="Connection settings for the CalDAV adapter.", + fields=( + Field("url", "string", "The account's CalDAV calendar home set. A trailing `/` is added " + "when missing.", required=True, json_type="string", coerce=_exact_str, + empty_is_missing=True), + Field("username", "string", "The login name, which can differ from the user id in the URL.", + required=True, json_type="string", coerce=_exact_str, empty_is_missing=True), + Field("app_password", "string", "An app password is recommended over the account password.", + required=True, json_type="string", coerce=_exact_str, empty_is_missing=True), + Field("inbox_list", "string", "The name of the one inbox list.", + required=True, json_type="string", coerce=_exact_str, empty_is_missing=True), + Field("someday_list", "string", "The name of the one someday/maybe list.", + required=True, json_type="string", coerce=_exact_str, empty_is_missing=True), + Field("exclude_lists", "array of strings", "Existing plain task lists that are not " + "projects.", default=[], json_type="array", coerce=_str_list), + ), + examples=({ + "url": "https:///remote.php/dav/calendars//", "username": "", + "app_password": "...", "inbox_list": "Inbox", "someday_list": "Someday", + "exclude_lists": ["..."], + },), + unknown_keys=REJECT, + unknown_text="Rejected.", +) + +TASKS = Shape( + key=config.TASKS_CONFIG_FILENAME, + title="Task tracker", + purpose="Picks the task tracker the weekly review reads and writes through, its connection " + "settings and the review's thresholds.", + fields=( + _SCHEMA, + Field("provider", "string", "Which adapter the review goes through. The provider's own " + "settings sit in a section of the same name.", + required=True, json_type="string", coerce=str), + Field( + "thresholds", "object", "The weekly review's three thresholds.", + required=True, json_type="object", + fields=( + _positive_integer("stalled_waiting_days", "A waiting-for item older than this many " + "days counts as overdue.", required=True), + _positive_integer("unpaged_project_weeks", "A tracker project with no `kb/` page is " + "reported once it is older than this many weeks.", required=True), + _positive_integer("someday_stale_months", "A someday/maybe item untouched for this " + "many months counts as stale.", required=True), + ), + ), + ), + section_key="provider", + sections=(SUPERPRODUCTIVITY, CALDAV), + examples=({ + "schema": 1, "provider": "superproductivity", + "thresholds": {"stalled_waiting_days": 14, "unpaged_project_weeks": 3, "someday_stale_months": 5}, + "superproductivity": {"access": "api", "api_base_url": "http://127.0.0.1:3876", "api_token": "..."}, + },), + absent="No tracker is configured - a valid state. The weekly review has nothing to check " + "against.", + malformed="An error, and `doctor` reports `FAIL`. A broken file is never read as \"no tracker " + "configured\".", + unknown_text="Ignored, so the section of a provider not chosen right now can stay in the file.", +) + +SHAPES = (TELEMETRY, REMOTES, UPLOAD, TASKS) + + +# --------------------------------------------------------------------------- +# Checking a value against a record +# --------------------------------------------------------------------------- + +_JSON_TYPES: dict[str, Callable[[Any], bool]] = { + "boolean": lambda v: isinstance(v, bool), + "integer": lambda v: isinstance(v, int) and not isinstance(v, bool), + "string": lambda v: isinstance(v, str), + "array": lambda v: isinstance(v, list), + "object": lambda v: isinstance(v, dict), +} + +_TYPE_NAMES = {bool: "a boolean", int: "an integer", float: "a number", str: "a string", + list: "an array", dict: "an object", type(None): "null"} + + +def _described(value: Any) -> str: + return _TYPE_NAMES.get(type(value), type(value).__name__) + + +def _article(noun: str) -> str: + return f"{'an' if noun[:1] in 'aeiou' else 'a'} {noun}" + + +def _check_fields( + shape: Shape, fields: tuple[Field, ...], data: dict, prefix: str, when: str, out: Checked, + values: dict, +) -> None: + for spec in fields: + path = prefix + spec.name + raw = data.get(spec.name, _ABSENT) + if raw is _ABSENT or (spec.empty_is_missing and not raw): + if spec.required: + why = f" - {spec.required_why}" if spec.required_why else "" + empty = " or empty" if spec.empty_is_missing else "" + out.problems.append(Problem(path, f"{path} is missing{empty}, and required{when}{why}")) + else: + values[spec.name] = spec.default + continue + if spec.fields: + if not isinstance(raw, dict): + out.problems.append(Problem(path, f"{path} must be an object, got {_described(raw)}")) + continue + nested: dict = {} + before = len(out.problems) + _check_fields(shape, spec.fields, raw, f"{path}.", when, out, nested) + if len(out.problems) == before: + values[spec.name] = nested + out.given.add(path) + continue + try: + value = spec.coerce(raw) if spec.coerce is not None else raw + except (TypeError, ValueError): + out.problems.append(Problem(path, f"{path} must be {_article(spec.type)}, got {_described(raw)}")) + continue + choices = shape.choices(spec) + if choices and value not in choices: + why = f" - {spec.required_why}" if spec.required_why else "" + out.problems.append(Problem( + path, f"{path} must be one of {', '.join(repr(c) for c in choices)}{why}; got {raw!r}" + )) + continue + problem = spec.check(value) if spec.check is not None else None + if problem: + out.problems.append(Problem(path, f"{path} {problem}")) + continue + if spec.json_type is not None and not _JSON_TYPES[spec.json_type](raw): + out.coerced.append(Problem( + path, f"{path} is {_described(raw)}, not a JSON {spec.json_type}, and is read as " + f"{value!r}", + )) + values[spec.name] = value + out.given.add(path) + + +def check(shape: Shape, data: Any) -> Checked: + """`data` read against `shape`: coerced values, problems, and what was + only accepted leniently. Never raises - each loader decides what a problem + means for it.""" + out = Checked() + if not isinstance(data, dict): + out.problems.append(Problem("", f"must contain a JSON object, not {_described(data)}")) + return out + _check_fields(shape, shape.fields, data, "", "", out, out.values) + known = {f.name for f in shape.fields} + + if shape.discriminator: + chosen = out.values.get(shape.discriminator) + variant = shape.variant_fields(chosen) if shape.discriminator in out.given else None + if variant is None: + # The discriminator is already a problem; an "unknown key" on top + # would only name fields of a variant nobody picked. + return out + _check_fields(shape, variant, data, "", f" when {shape.discriminator}: {chosen!r}", out, + out.values) + known |= {f.name for f in variant} + + if shape.section_key: + known |= {s.key for s in shape.sections} + chosen = out.values.get(shape.section_key) + if shape.section_key in out.given and not isinstance(data.get(chosen), dict): + out.problems.append(Problem( + chosen, f"missing or non-object {chosen!r} section holding that provider's own " + "connection settings", + )) + + out.unknown = sorted(set(data) - known) + if out.unknown and shape.unknown_keys == REJECT: + why = f" - {shape.unknown_why}" if shape.unknown_why else "" + out.problems.append(Problem("", f"does not take {out.unknown}{why}")) + return out + + +def deviations(shape: Shape, data: Any) -> list[str]: + """Everything in `data` that a strict reading of `shape` would not take as + written: problems, values accepted only by coercion, and unknown keys. For + a file whose loader is silent about all three - telemetry - this is what + `doctor` reports.""" + checked = check(shape, data) + found = [p.message for p in checked.problems] + [p.message for p in checked.coerced] + if shape.unknown_keys == IGNORE: + found += [f"unknown key {key!r} is ignored" for key in checked.unknown] + return found + + +# --------------------------------------------------------------------------- +# Rendering +# --------------------------------------------------------------------------- + + +def render_expected(shape: Shape) -> str: + """The `Expected:` text of a loader's error message: the record's example, + or its examples joined by "or" where a discriminator offers several.""" + return " or ".join(json.dumps(example) for example in shape.examples) + + +def _cell(text: str) -> str: + return text.replace("|", "\\|").replace("\n", " ") + + +def _default_cell(spec: Field) -> str: + if spec.required or spec.fields: + return "-" + if spec.default_text: + return spec.default_text + if spec.default is None: + return "-" + return f"`{json.dumps(spec.default)}`" + + +def _rows(fields: tuple[Field, ...], prefix: str = "") -> list[str]: + rows = [] + for spec in fields: + path = prefix + spec.name + required = "yes" if spec.required else "no" + if spec.required and spec.empty_is_missing: + required = "yes, not empty" + description = " ".join(part for part in (spec.description, spec.lenient) if part) + rows.append( + f"| `{path}` | {spec.type} | {required} | {_cell(_default_cell(spec))} | {_cell(description)} |" + ) + rows += _rows(spec.fields, f"{path}.") + return rows + + +def _table(fields: tuple[Field, ...]) -> list[str]: + return ["| Field | Type | Required | Default | Description |", + "|-------|------|----------|---------|-------------|", *_rows(fields)] + + +def _example(example: dict) -> list[str]: + return ["```json", json.dumps(example, indent=2), "```"] + + +def _render_section(shape: Shape) -> list[str]: + lines = [f"**The `{shape.key}` section.** {shape.purpose}", "", *_table(shape.fields), ""] + for value, fields in shape.variants: + lines += [f"With `{shape.discriminator}: \"{value}\"`:", "", *_table(fields), ""] + lines += [f"- **Unknown keys:** {shape.unknown_text}", ""] + for example in shape.examples: + lines += [*_example({shape.key: example}), ""] + return lines + + +def render_markdown(shape: Shape) -> str: + """The reference text for one file: its fields, what an absent and a + malformed file mean, what happens to unknown keys, and an example.""" + lines = [*_table(shape.fields), ""] + lines += [ + f"- **Absent:** {shape.absent}", + f"- **Malformed:** {shape.malformed}", + f"- **Unknown keys:** {shape.unknown_text}", + "", + ] + for section in shape.sections: + lines += _render_section(section) + lines += ["A complete example:" if shape.sections else "Example:", ""] + for example in shape.examples: + lines += [*_example(example), ""] + return "\n".join(lines).rstrip("\n") + "\n" diff --git a/tools/chemenu/reference_doc.py b/tools/chemenu/reference_doc.py index a0d2244..d3d7026 100644 --- a/tools/chemenu/reference_doc.py +++ b/tools/chemenu/reference_doc.py @@ -27,6 +27,11 @@ pages describe the stack's shipped defaults and are `origin_only`: generated and checked in the origin repo, where `types/*.md` is exactly what ships as `.template`, and left alone in an instance - one carrying a release stamp, the same line the telemetry default draws. + +`docs/reference/configuration.md` carries four more regions, one per +hand-written configuration file, rendered from the records in `config_shapes`. +Those records are the tool's own, read by the loaders and shipped with +`tools/`, so their regions are generated and checked in an instance too. """ from __future__ import annotations @@ -34,7 +39,7 @@ import re from pathlib import Path from typing import Callable, NamedTuple, Optional -from chemenu import blocks, cli_contract, config, toc +from chemenu import blocks, cli_contract, config, config_shapes, toc from chemenu import version as version_mod REFERENCE_DIR = "docs/reference" @@ -43,6 +48,7 @@ COMMANDS_INDEX = f"{COMMANDS_DIR}/README.md" EXIT_CODES = f"{REFERENCE_DIR}/exit-codes.md" SKILLS = f"{REFERENCE_DIR}/skills.md" PAGE_TYPES = f"{REFERENCE_DIR}/page-types.md" +CONFIGURATION = f"{REFERENCE_DIR}/configuration.md" COMMANDS_REGION = "reference-commands" INDEX_REGION = "reference-commands-index" @@ -50,6 +56,18 @@ EXIT_CODES_REGION = "reference-exit-codes" SKILLS_REGION = "reference-skills" PAGE_TYPES_REGION = "reference-page-types" +# One region per hand-written configuration file, each placed inside the +# section of `configuration.md` that carries the file's behaviour in prose. +# Rendered from the `config_shapes` record named beside it, which ships with +# `tools/`, so unlike the two pages above they are generated and checked in an +# instance as well. +CONFIG_REGIONS = ( + ("reference-config-telemetry", "TELEMETRY"), + ("reference-config-remotes", "REMOTES"), + ("reference-config-tasks", "TASKS"), + ("reference-config-upload", "UPLOAD"), +) + _SLUG_RE = re.compile(r"[^a-z0-9]+") @@ -408,6 +426,12 @@ def regions( # its own skills and types for the real ones. found.append(Region(SKILLS, SKILLS_REGION, lambda: render_skills(shipped_skills()), origin_only=True)) found.append(Region(PAGE_TYPES, PAGE_TYPES_REGION, lambda: render_page_types(page_types()), origin_only=True)) + # The record is looked up at render time, so a test replacing one sees it. + found += [ + Region(CONFIGURATION, name, + lambda attr=shape_name: config_shapes.render_markdown(getattr(config_shapes, attr))) + for name, shape_name in CONFIG_REGIONS + ] return found @@ -422,7 +446,10 @@ def refresh(text: str, region: Region) -> Optional[str]: if blocks.find(text, region.name) is None: return None content = region.render().strip("\n") - wrapped = blocks.open_marker(region.name) + "\n" + content + "\n" + blocks.close_marker(region.name) + # The trailing newline keeps a blank line between the region and prose + # that follows it; `blocks.replace` was written for regions that end a + # page, and the configuration page is the first to carry them mid-page. + wrapped = blocks.open_marker(region.name) + "\n" + content + "\n" + blocks.close_marker(region.name) + "\n" return toc.upsert(blocks.replace(text, region.name, wrapped)) diff --git a/tools/chemenu/tasks/caldav.py b/tools/chemenu/tasks/caldav.py index 1fec3d7..4befb42 100644 --- a/tools/chemenu/tasks/caldav.py +++ b/tools/chemenu/tasks/caldav.py @@ -82,6 +82,7 @@ import icalendar import urllib.error import urllib.request +from chemenu import config_shapes from chemenu.errors import ValidationError from chemenu.tasks.protocol import ( OpenItem, @@ -102,16 +103,6 @@ WAITING_CATEGORY = "waiting" SOURCE_KIND = "api" -_EXPECTED_CONFIG = ( - '{"url": "https:///remote.php/dav/calendars//", "username": "", ' - '"app_password": "...", "inbox_list": "Inbox", "someday_list": "Someday", ' - '"exclude_lists": ["..."]} - see INSTALL.md' -) - -_REQUIRED_STRING_FIELDS = ("url", "username", "app_password", "inbox_list", "someday_list") -_KNOWN_FIELDS = frozenset({*_REQUIRED_STRING_FIELDS, "exclude_lists"}) - - @dataclass(frozen=True) class CalDAVConfig: """This provider's own section of `.wikitool-tasks.json` @@ -129,40 +120,21 @@ class CalDAVConfig: @classmethod def from_dict(cls, data: dict[str, Any]) -> "CalDAVConfig": - if not isinstance(data, dict): - raise ValidationError(f"caldav config must be an object. Expected: {_EXPECTED_CONFIG}") - - missing = [ - key for key in _REQUIRED_STRING_FIELDS - if not isinstance(data.get(key), str) or not data[key] - ] - if missing: + shape = config_shapes.CALDAV + checked = config_shapes.check(shape, data) + if checked.problems: raise ValidationError( - f"caldav config is missing or has an empty {', '.join(missing)}. " - f"Expected: {_EXPECTED_CONFIG}" + f"caldav config: {checked.message()}. Expected: {config_shapes.render_expected(shape)}" ) - - extra = sorted(set(data) - _KNOWN_FIELDS) - if extra: - raise ValidationError( - f"caldav config does not take {extra} - unexpected field(s). Expected: {_EXPECTED_CONFIG}" - ) - - exclude_lists = data.get("exclude_lists", []) - if not isinstance(exclude_lists, list) or not all(isinstance(x, str) for x in exclude_lists): - raise ValidationError( - f"caldav config: 'exclude_lists' must be a list of strings if given. " - f"Expected: {_EXPECTED_CONFIG}" - ) - - url = data["url"] if data["url"].endswith("/") else data["url"] + "/" + values = checked.values + url = values["url"] if values["url"].endswith("/") else values["url"] + "/" return cls( url=url, - username=data["username"], - app_password=data["app_password"], - inbox_list=data["inbox_list"], - someday_list=data["someday_list"], - exclude_lists=tuple(exclude_lists), + username=values["username"], + app_password=values["app_password"], + inbox_list=values["inbox_list"], + someday_list=values["someday_list"], + exclude_lists=tuple(values["exclude_lists"]), ) diff --git a/tools/chemenu/tasks/config.py b/tools/chemenu/tasks/config.py index 46dba48..bbbd14f 100644 --- a/tools/chemenu/tasks/config.py +++ b/tools/chemenu/tasks/config.py @@ -12,14 +12,14 @@ from __future__ import annotations from dataclasses import dataclass from typing import Any -from chemenu import config +from chemenu import config, config_shapes from chemenu.errors import ValidationError -# The providers this package ships an adapter for. Checked at read time so a -# typo in `provider` fails here, at the one place that knows the full list, -# rather than surfacing later as an unhelpful "unknown provider" from whatever -# code tried to dispatch on it. -KNOWN_PROVIDERS = ("superproductivity", "caldav") +# The providers this package ships an adapter for: the sections of the file's +# record (`config_shapes.TASKS`). Checked at read time so a typo in `provider` +# fails here, rather than surfacing later as an unhelpful "unknown provider" +# from whatever code tried to dispatch on it. +KNOWN_PROVIDERS = tuple(section.key for section in config_shapes.TASKS.sections) @dataclass(frozen=True) @@ -67,56 +67,16 @@ def read_config(root: "Any") -> "TasksConfig | None": "tracker the weekly review talks to, so a broken file is not treated as 'no " "tracker configured' - fix it or delete it deliberately." ) from exc - if not isinstance(data, dict): - raise ValidationError(f"{label} must contain a JSON object.") - - expected = ( - '{"schema": 1, "provider": "superproductivity", ' - '"thresholds": {"stalled_waiting_days": 14, "unpaged_project_weeks": 3, ' - '"someday_stale_months": 5}, "superproductivity": {"access": "api", ' - '"api_base_url": "http://127.0.0.1:3876", "api_token": "..."} ' - '(or {"access": "snapshot", "backups_dir": "..."} - see INSTALL.md)}' - ) - try: - provider = str(data["provider"]) - raw_thresholds = data["thresholds"] - stalled_waiting_days = int(raw_thresholds["stalled_waiting_days"]) - unpaged_project_weeks = int(raw_thresholds["unpaged_project_weeks"]) - someday_stale_months = int(raw_thresholds["someday_stale_months"]) - except (KeyError, TypeError, ValueError) as exc: + shape = config_shapes.TASKS + checked = config_shapes.check(shape, data) + if checked.problems: raise ValidationError( - f"{label} is missing or misshapes a required field ({exc}). " - f"Expected: {expected}" - ) from exc - - if provider not in KNOWN_PROVIDERS: - raise ValidationError( - f"{label}: unknown provider {provider!r}. " - f"Known: {', '.join(KNOWN_PROVIDERS)}" + f"{label}: {checked.message()}. Expected: {config_shapes.render_expected(shape)}" ) - for field_name, value in ( - ("stalled_waiting_days", stalled_waiting_days), - ("unpaged_project_weeks", unpaged_project_weeks), - ("someday_stale_months", someday_stale_months), - ): - if value <= 0: - raise ValidationError( - f"{label}: thresholds.{field_name} must be positive." - ) - - provider_config = data.get(provider) - if not isinstance(provider_config, dict): - raise ValidationError( - f"{label}: missing or non-object {provider!r} section " - f"holding that provider's own connection settings. Expected: {expected}" - ) - + provider = checked.values["provider"] + thresholds = checked.values["thresholds"] return TasksConfig( provider=provider, - thresholds=Thresholds( - stalled_waiting_days=stalled_waiting_days, - unpaged_project_weeks=unpaged_project_weeks, - someday_stale_months=someday_stale_months, - ), - provider_config=provider_config, + thresholds=Thresholds(**thresholds), + provider_config=data[provider], ) diff --git a/tools/chemenu/tasks/superproductivity.py b/tools/chemenu/tasks/superproductivity.py index c97fef6..0aa86ce 100644 --- a/tools/chemenu/tasks/superproductivity.py +++ b/tools/chemenu/tasks/superproductivity.py @@ -125,6 +125,7 @@ from datetime import date, datetime, timezone from pathlib import Path from typing import Any, Optional +from chemenu import config_shapes from chemenu.errors import HumanInterventionRequired, ValidationError from chemenu.tasks.protocol import ( OpenItem, @@ -150,11 +151,13 @@ WAITING_TAG_TITLE = "waiting" # this module ever writes it. INBOX_PROJECT_ID = "INBOX_PROJECT" -DEFAULT_API_BASE_URL = "http://127.0.0.1:3876" +# This section's shape - its fields per `access`, and these values - is the +# record `config_shapes.SUPERPRODUCTIVITY`. +DEFAULT_API_BASE_URL = config_shapes.SUPERPRODUCTIVITY.field("api_base_url").default ACCESS_API = "api" ACCESS_SNAPSHOT = "snapshot" -KNOWN_ACCESS = (ACCESS_API, ACCESS_SNAPSHOT) +KNOWN_ACCESS = tuple(value for value, _ in config_shapes.SUPERPRODUCTIVITY.variants) # `electron/backup.ts` writes exactly this shape - a fixed-width timestamp, # no prefix. Restricting the glob to it (Gitea #133) is what keeps a manually @@ -165,10 +168,6 @@ _SNAPSHOT_GLOB = ( "[0-9][0-9][0-9][0-9][0-9][0-9].json" ) -_EXPECTED_API_CONFIG = '{"access": "api", "api_base_url": "http://127.0.0.1:3876", "api_token": "..."}' -_EXPECTED_SNAPSHOT_CONFIG = '{"access": "snapshot", "backups_dir": "~/.config/superProductivity/backups"}' - - @dataclass(frozen=True) class SuperProductivityConfig: """This provider's own section of `.wikitool-tasks.json` @@ -185,53 +184,22 @@ class SuperProductivityConfig: @classmethod def from_dict(cls, data: dict[str, Any]) -> "SuperProductivityConfig": - if not isinstance(data, dict): + shape = config_shapes.SUPERPRODUCTIVITY + checked = config_shapes.check(shape, data) + if checked.problems: raise ValidationError( - "superproductivity config must be an object. Expected one of: " - f"{_EXPECTED_API_CONFIG} or {_EXPECTED_SNAPSHOT_CONFIG}" + f"superproductivity config: {checked.message()}. " + f"Expected one of: {config_shapes.render_expected(shape)}" ) - access = data.get("access") - if access not in KNOWN_ACCESS: - raise ValidationError( - "superproductivity config needs 'access', either 'api' or 'snapshot' - " - "required, no default and no fallback between them (Gitea #133). Got: " - f"{access!r}. Expected one of: {_EXPECTED_API_CONFIG} or {_EXPECTED_SNAPSHOT_CONFIG}" - ) - - if access == ACCESS_API: - extra = sorted(set(data) - {"access", "api_base_url", "api_token"}) - if extra: - raise ValidationError( - f"superproductivity config: access: 'api' does not take {extra} - a " - "section names only one access path's own fields (Gitea #133). " - f"Expected: {_EXPECTED_API_CONFIG}" - ) - api_token = data.get("api_token") - if not isinstance(api_token, str) or not api_token: - raise ValidationError( - "superproductivity config: api_token is required when access: 'api' - " - "every endpoint but GET /health requires Authorization: Bearer . " - f"Expected: {_EXPECTED_API_CONFIG}" - ) - api_base_url = str(data.get("api_base_url") or DEFAULT_API_BASE_URL) - return cls(access=access, backups_dir=None, api_base_url=api_base_url, api_token=api_token) - - extra = sorted(set(data) - {"access", "backups_dir"}) - if extra: - raise ValidationError( - f"superproductivity config: access: 'snapshot' does not take {extra} - a " - "section names only one access path's own fields (Gitea #133). " - f"Expected: {_EXPECTED_SNAPSHOT_CONFIG}" - ) - backups_dir = data.get("backups_dir") - if not backups_dir: - raise ValidationError( - "superproductivity config: backups_dir is required when access: 'snapshot' - " - f"the read path has nothing to read otherwise. Expected: {_EXPECTED_SNAPSHOT_CONFIG}" + values = checked.values + if values["access"] == ACCESS_API: + return cls( + access=ACCESS_API, backups_dir=None, + api_base_url=values["api_base_url"], api_token=values["api_token"], ) return cls( - access=access, - backups_dir=Path(backups_dir).expanduser(), + access=ACCESS_SNAPSHOT, + backups_dir=Path(values["backups_dir"]).expanduser(), api_base_url=DEFAULT_API_BASE_URL, api_token=None, ) diff --git a/tools/chemenu/telemetry/policy.py b/tools/chemenu/telemetry/policy.py index 8a0158d..b8443e1 100644 --- a/tools/chemenu/telemetry/policy.py +++ b/tools/chemenu/telemetry/policy.py @@ -33,10 +33,12 @@ import os from dataclasses import dataclass from pathlib import Path -from chemenu import config +from chemenu import config, config_shapes -DEFAULT_MAX_SESSION_BYTES = 5 * 1024 * 1024 -DEFAULT_KEEP_SESSIONS = 250 +# The file's shape - and these two defaults with it - is the record in +# `chemenu.config_shapes`, which is stdlib-only for exactly this import. +DEFAULT_MAX_SESSION_BYTES = config_shapes.TELEMETRY.field("max_session_bytes").default +DEFAULT_KEEP_SESSIONS = config_shapes.TELEMETRY.field("keep_sessions").default ENV_ENABLED = "WIKI_TRACE" ENV_MAX_SESSION_BYTES = "WIKI_TRACE_MAX_SESSION_BYTES" @@ -76,6 +78,12 @@ def _positive_int(value: object) -> int | None: return parsed if parsed > 0 else None +def _given(checked: "config_shapes.Checked", name: str) -> int | None: + """A cap the file sets and the record accepts, else None - so the + environment and the default decide, as for an absent file.""" + return checked.values[name] if name in checked.given else None + + def _read_config(root: Path) -> dict: try: raw = (root / config.TELEMETRY_FILENAME).read_text(encoding="utf-8") @@ -101,8 +109,10 @@ def _root_defaults(root: Path) -> _RootDefaults: else f"{config.RELEASE_STAMP_FILENAME} present - distributed instance, default off" ) - data = _read_config(root) - file_enabled = bool(data["enabled"]) if "enabled" in data else None + # Silent on the hot path: a value the record does not accept is dropped, + # never an error. `doctor` reports it (`config_shapes.deviations`). + checked = config_shapes.check(config_shapes.TELEMETRY, _read_config(root)) + file_enabled = checked.values.get("enabled") file_reason = ( f"{config.TELEMETRY_FILENAME} sets enabled={file_enabled}" if file_enabled is not None @@ -114,8 +124,8 @@ def _root_defaults(root: Path) -> _RootDefaults: form_reason=form_reason, file_enabled=file_enabled, file_reason=file_reason, - file_max_bytes=_positive_int(data.get("max_session_bytes")), - file_keep_sessions=_positive_int(data.get("keep_sessions")), + file_max_bytes=_given(checked, "max_session_bytes"), + file_keep_sessions=_given(checked, "keep_sessions"), ) _cache[root] = result return result diff --git a/tools/chemenu/tests/test_config_shapes.py b/tools/chemenu/tests/test_config_shapes.py new file mode 100644 index 0000000..48a774f --- /dev/null +++ b/tools/chemenu/tests/test_config_shapes.py @@ -0,0 +1,608 @@ +"""`chemenu.config_shapes` - one record per hand-written configuration file, +which the loaders read and `docs reference` renders. + +The drop-in half compares every loader against a copy of the loader as it stood +before the records existed (the `_legacy_*` oracles below, kept verbatim in +what they accept): any file it accepted must still be accepted with the same +values, and any file it rejected must still be rejected. Only the wording of a +rejection may differ. +""" +from __future__ import annotations + +import ast +import dataclasses +import json +import re +import sys +from pathlib import Path + +import pytest +import typer +from typer.testing import CliRunner + +from chemenu import blocks, config, config_shapes, reference_doc +from chemenu.cli import app +from chemenu.commands import docs_verify, git_publish +from chemenu.errors import ValidationError +from chemenu.tasks import caldav, superproductivity +from chemenu.tasks import config as tasks_config +from chemenu.telemetry import policy + +runner = CliRunner() + +PACKAGE = Path(config_shapes.__file__).parent + + +# --- the module itself ------------------------------------------------------- + + +def test_the_module_imports_only_the_stdlib_and_chemenu_config(): + """A hook handler imports `chemenu.telemetry` on every tool call without + the venv, and the telemetry policy imports this module.""" + tree = ast.parse(Path(config_shapes.__file__).read_text(encoding="utf-8")) + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported |= {alias.name for alias in node.names} + elif isinstance(node, ast.ImportFrom): + if node.module == "chemenu": + imported |= {f"chemenu.{alias.name}" for alias in node.names} + elif node.module != "__future__": + imported.add(node.module) + foreign = { + name for name in imported + if name != "chemenu.config" and name.split(".")[0] not in sys.stdlib_module_names + } + assert foreign == set() + + +def test_the_telemetry_policy_reads_the_record(): + assert policy.config_shapes is config_shapes + assert policy.DEFAULT_MAX_SESSION_BYTES == config_shapes.TELEMETRY.field("max_session_bytes").default + assert policy.DEFAULT_KEEP_SESSIONS == config_shapes.TELEMETRY.field("keep_sessions").default + + +def test_one_record_per_file(): + assert [shape.key for shape in config_shapes.SHAPES] == [ + config.TELEMETRY_FILENAME, config.PUBLISH_REMOTES_FILENAME, + config.UPLOAD_CONFIG_FILENAME, config.TASKS_CONFIG_FILENAME, + ] + + +LOADERS = ("telemetry/policy.py", "commands/git_publish.py", "upload.py", "tasks/config.py", + "tasks/superproductivity.py", "tasks/caldav.py") + + +def test_no_hand_written_expected_text_is_left_in_a_loader(): + pattern = re.compile(r"_EXPECTED_\w*\s*=|\bexpected\s*=\s*\(|Expected: \{\\?\"") + offenders = [ + relative for relative in LOADERS + if pattern.search((PACKAGE / relative).read_text(encoding="utf-8")) + ] + assert offenders == [] + + +def test_the_known_names_follow_the_records(): + assert tasks_config.KNOWN_PROVIDERS == ("superproductivity", "caldav") + assert superproductivity.KNOWN_ACCESS == ("api", "snapshot") + + +# --- the legacy oracles: the loaders as they were before the records -------- + + +def _legacy_upload(data): + if not isinstance(data, dict): + raise ValidationError("not an object") + try: + identity_header = str(data.get("identity_header") or "X-Forwarded-User") + max_bytes = int(data["max_bytes"]) + extensions = tuple(sorted({str(ext).lower() for ext in data["allowed_extensions"]})) + quota = data["quota"] + submissions_per_day = int(quota["submissions_per_day"]) + bytes_per_day = int(quota["bytes_per_day"]) + except (KeyError, TypeError, ValueError) as exc: + raise ValidationError("misshaped") from exc + if max_bytes <= 0 or submissions_per_day <= 0 or bytes_per_day <= 0: + raise ValidationError("not positive") + if not extensions or [ext for ext in extensions if not ext.startswith(".")]: + raise ValidationError("extensions") + return (identity_header, max_bytes, extensions, submissions_per_day, bytes_per_day) + + +def _legacy_tasks(data): + if not isinstance(data, dict): + raise ValidationError("not an object") + try: + provider = str(data["provider"]) + raw = data["thresholds"] + thresholds = (int(raw["stalled_waiting_days"]), int(raw["unpaged_project_weeks"]), + int(raw["someday_stale_months"])) + except (KeyError, TypeError, ValueError) as exc: + raise ValidationError("misshaped") from exc + if provider not in ("superproductivity", "caldav"): + raise ValidationError("provider") + if any(value <= 0 for value in thresholds): + raise ValidationError("not positive") + if not isinstance(data.get(provider), dict): + raise ValidationError("section") + return (provider, thresholds, data.get(provider)) + + +def _legacy_sp(data): + if not isinstance(data, dict): + raise ValidationError("not an object") + access = data.get("access") + if access not in ("api", "snapshot"): + raise ValidationError("access") + if access == "api": + if set(data) - {"access", "api_base_url", "api_token"}: + raise ValidationError("extra") + token = data.get("api_token") + if not isinstance(token, str) or not token: + raise ValidationError("token") + return ("api", None, str(data.get("api_base_url") or "http://127.0.0.1:3876"), token) + if set(data) - {"access", "backups_dir"}: + raise ValidationError("extra") + if not data.get("backups_dir"): + raise ValidationError("backups_dir") + return ("snapshot", Path(data["backups_dir"]).expanduser(), "http://127.0.0.1:3876", None) + + +_CALDAV_STRINGS = ("url", "username", "app_password", "inbox_list", "someday_list") + + +def _legacy_caldav(data): + if not isinstance(data, dict): + raise ValidationError("not an object") + if [k for k in _CALDAV_STRINGS if not isinstance(data.get(k), str) or not data[k]]: + raise ValidationError("missing") + if set(data) - {*_CALDAV_STRINGS, "exclude_lists"}: + raise ValidationError("extra") + exclude = data.get("exclude_lists", []) + if not isinstance(exclude, list) or not all(isinstance(x, str) for x in exclude): + raise ValidationError("exclude_lists") + url = data["url"] if data["url"].endswith("/") else data["url"] + "/" + return (url, data["username"], data["app_password"], data["inbox_list"], data["someday_list"], + tuple(exclude)) + + +def _legacy_remotes(data): + urls = data.get("allowed_push_urls") + if not isinstance(urls, list) or not all(isinstance(u, str) for u in urls): + raise ValidationError("urls") + return urls + + +def _legacy_telemetry(data): + def positive(value): + try: + parsed = int(value) + except (TypeError, ValueError): + return None + return parsed if parsed > 0 else None + + data = data if isinstance(data, dict) else {} + return ( + bool(data["enabled"]) if "enabled" in data else None, + positive(data.get("max_session_bytes")), + positive(data.get("keep_sessions")), + ) + + +# --- the loaders today, reduced to the same tuples -------------------------- + + +def _upload(data, tmp_path): + (tmp_path / config.UPLOAD_CONFIG_FILENAME).write_text(json.dumps(data), encoding="utf-8") + from chemenu import upload + + cfg = upload.read_config(tmp_path) + return (cfg.identity_header, cfg.max_bytes, cfg.allowed_extensions, cfg.submissions_per_day, + cfg.bytes_per_day) + + +def _tasks(data, tmp_path): + (tmp_path / config.TASKS_CONFIG_FILENAME).write_text(json.dumps(data), encoding="utf-8") + cfg = tasks_config.read_config(tmp_path) + t = cfg.thresholds + return (cfg.provider, (t.stalled_waiting_days, t.unpaged_project_weeks, t.someday_stale_months), + cfg.provider_config) + + +def _sp(data, tmp_path): + cfg = superproductivity.SuperProductivityConfig.from_dict(data) + return (cfg.access, cfg.backups_dir, cfg.api_base_url, cfg.api_token) + + +def _caldav(data, tmp_path): + cfg = caldav.CalDAVConfig.from_dict(data) + return (cfg.url, cfg.username, cfg.app_password, cfg.inbox_list, cfg.someday_list, + cfg.exclude_lists) + + +def _remotes(data, tmp_path): + (tmp_path / config.PUBLISH_REMOTES_FILENAME).write_text(json.dumps(data), encoding="utf-8") + try: + return git_publish.read_allowed_push_urls() + except typer.Exit as exc: + raise ValidationError("fail()") from exc + + +def _telemetry(data, tmp_path): + (tmp_path / config.TELEMETRY_FILENAME).write_text(json.dumps(data), encoding="utf-8") + policy.reset_cache() + defaults = policy._root_defaults(tmp_path.resolve()) + policy.reset_cache() + return (defaults.file_enabled, defaults.file_max_bytes, defaults.file_keep_sessions) + + +UPLOAD_OK = { + "schema": 1, "identity_header": "X-Forwarded-User", "max_bytes": 10485760, + "allowed_extensions": [".md", ".txt", ".pdf", ".html", ".csv", ".json", ".png", ".jpg"], + "quota": {"submissions_per_day": 20, "bytes_per_day": 52428800}, +} +TASKS_OK = { + "schema": 1, "provider": "superproductivity", + "thresholds": {"stalled_waiting_days": 14, "unpaged_project_weeks": 3, "someday_stale_months": 5}, + "superproductivity": {"access": "api", "api_base_url": "http://127.0.0.1:3876", "api_token": "t"}, +} +CALDAV_OK = { + "url": "https://host/remote.php/dav/calendars/me", "username": "me", "app_password": "pw", + "inbox_list": "Inbox", "someday_list": "Someday", "exclude_lists": ["Shopping"], +} + + +def _with(base, **changes): + data = json.loads(json.dumps(base)) + for key, value in changes.items(): + target = data + *parents, leaf = key.split("__") + for parent in parents: + target = target[parent] + if value is _DROP: + del target[leaf] + else: + target[leaf] = value + return data + + +_DROP = object() + +CASES = [ + # upload: the documented file, then every tolerance and every rejection + ("upload", UPLOAD_OK), + ("upload", _with(UPLOAD_OK, schema=_DROP)), + ("upload", _with(UPLOAD_OK, identity_header=_DROP)), + ("upload", _with(UPLOAD_OK, identity_header="")), + ("upload", _with(UPLOAD_OK, max_bytes="10485760")), + ("upload", _with(UPLOAD_OK, quota__bytes_per_day="100")), + ("upload", _with(UPLOAD_OK, allowed_extensions=[".MD", ".md", ".Pdf"])), + ("upload", _with(UPLOAD_OK, surplus="ignored")), + ("upload", _with(UPLOAD_OK, quota__surplus=1)), + ("upload", _with(UPLOAD_OK, max_bytes=0)), + ("upload", _with(UPLOAD_OK, max_bytes="ten")), + ("upload", _with(UPLOAD_OK, max_bytes=_DROP)), + ("upload", _with(UPLOAD_OK, allowed_extensions=["md"])), + ("upload", _with(UPLOAD_OK, allowed_extensions=[])), + ("upload", _with(UPLOAD_OK, allowed_extensions=".md")), + ("upload", _with(UPLOAD_OK, quota=[1, 2])), + ("upload", _with(UPLOAD_OK, quota__submissions_per_day=-1)), + ("upload", [1, 2]), + # tasks + ("tasks", TASKS_OK), + ("tasks", _with(TASKS_OK, schema=_DROP)), + ("tasks", _with(TASKS_OK, thresholds__stalled_waiting_days="14")), + ("tasks", _with(TASKS_OK, caldav={"url": "unused"})), + ("tasks", _with(TASKS_OK, surplus=True)), + ("tasks", _with(TASKS_OK, thresholds__surplus=1)), + ("tasks", _with(TASKS_OK, provider="asana")), + ("tasks", _with(TASKS_OK, provider=_DROP)), + ("tasks", _with(TASKS_OK, thresholds__someday_stale_months=0)), + ("tasks", _with(TASKS_OK, thresholds__unpaged_project_weeks=_DROP)), + ("tasks", _with(TASKS_OK, thresholds="soon")), + ("tasks", _with(TASKS_OK, superproductivity=_DROP)), + ("tasks", _with(TASKS_OK, superproductivity="api")), + ("tasks", _with(TASKS_OK, provider="caldav")), + ("tasks", "not an object"), + # the Super Productivity section + ("sp", {"access": "api", "api_base_url": "http://127.0.0.1:3876", "api_token": "t"}), + ("sp", {"access": "api", "api_token": "t"}), + ("sp", {"access": "api", "api_token": "t", "api_base_url": ""}), + ("sp", {"access": "snapshot", "backups_dir": "~/.config/superProductivity/backups"}), + ("sp", {"access": "api"}), + ("sp", {"access": "api", "api_token": ""}), + ("sp", {"access": "api", "api_token": 7}), + ("sp", {"access": "api", "api_token": "t", "backups_dir": "/x"}), + ("sp", {"access": "snapshot", "backups_dir": "/x", "api_token": "t"}), + ("sp", {"access": "snapshot"}), + ("sp", {"access": "snapshot", "backups_dir": ""}), + ("sp", {"access": "auto", "backups_dir": "/x"}), + ("sp", {"backups_dir": "/x"}), + ("sp", ["api"]), + # the CalDAV section + ("caldav", CALDAV_OK), + ("caldav", _with(CALDAV_OK, exclude_lists=_DROP)), + ("caldav", _with(CALDAV_OK, url="https://host/dav/")), + ("caldav", _with(CALDAV_OK, url=_DROP)), + ("caldav", _with(CALDAV_OK, username="")), + ("caldav", _with(CALDAV_OK, app_password=5)), + ("caldav", _with(CALDAV_OK, exclude_lists="Shopping")), + ("caldav", _with(CALDAV_OK, exclude_lists=[1])), + ("caldav", _with(CALDAV_OK, backups_dir="/x")), + # remotes + ("remotes", {"schema": 1, "allowed_push_urls": ["git@example.org:me/my-wiki.git"]}), + ("remotes", {"allowed_push_urls": []}), + ("remotes", {"allowed_push_urls": ["a"], "surplus": 1}), + ("remotes", {"allowed_push_urls": "not-a-list"}), + ("remotes", {"allowed_push_urls": [1]}), + ("remotes", {}), + # telemetry: never rejected, only read differently + ("telemetry", {"enabled": True, "max_session_bytes": 5242880, "keep_sessions": 250}), + ("telemetry", {}), + ("telemetry", {"enabled": "no"}), + ("telemetry", {"enabled": 0}), + ("telemetry", {"enabled": None}), + ("telemetry", {"max_session_bytes": "100", "keep_sessions": 3.7}), + ("telemetry", {"max_session_bytes": 0, "keep_sessions": -2}), + ("telemetry", {"max_session_bytes": "lots", "keep_sessions": [1]}), + ("telemetry", {"surplus": 1, "schema": 1}), + ("telemetry", [True]), +] + +ORACLES = { + "upload": (_legacy_upload, _upload), + "tasks": (_legacy_tasks, _tasks), + "sp": (_legacy_sp, _sp), + "caldav": (_legacy_caldav, _caldav), + "remotes": (_legacy_remotes, _remotes), + "telemetry": (_legacy_telemetry, _telemetry), +} + + +def _outcome(function, data, *args): + try: + return ("accepted", function(data, *args)) + except ValidationError: + return ("rejected", None) + + +@pytest.mark.parametrize("loader,data", CASES, ids=[f"{n}-{i}" for i, (n, _) in enumerate(CASES)]) +def test_every_loader_accepts_and_rejects_exactly_what_it_did_before(loader, data, tmp_path, monkeypatch): + legacy, current = ORACLES[loader] + monkeypatch.setattr(config, "ROOT", tmp_path) + assert _outcome(current, data, tmp_path) == _outcome(legacy, data) + + +def test_the_cases_cover_both_outcomes_for_every_loader(tmp_path): + for loader, (legacy, _) in ORACLES.items(): + if loader == "telemetry": + continue + outcomes = {_outcome(legacy, data)[0] for name, data in CASES if name == loader} + assert outcomes == {"accepted", "rejected"}, loader + + +def test_a_rejection_keeps_its_cause_and_renders_the_expected_text(tmp_path): + (tmp_path / config.UPLOAD_CONFIG_FILENAME).write_text( + json.dumps(_with(UPLOAD_OK, max_bytes=0)), encoding="utf-8" + ) + from chemenu import upload + + with pytest.raises(ValidationError) as caught: + upload.read_config(tmp_path) + message = str(caught.value) + assert "max_bytes must be positive" in message + assert f"Expected: {config_shapes.render_expected(config_shapes.UPLOAD)}" in message + + +# --- the loaders take their expectations from the record -------------------- + + +def _add_required(shape, name="added"): + return dataclasses.replace( + shape, fields=shape.fields + (config_shapes.Field(name, "string", "x", required=True),) + ) + + +def _not_required(shape, name, default): + fields = tuple( + dataclasses.replace(f, required=False, default=default) if f.name == name else f + for f in shape.fields + ) + return dataclasses.replace(shape, fields=fields) + + +def test_upload_follows_a_field_made_required_or_optional(tmp_path, monkeypatch): + from chemenu import upload + + (tmp_path / config.UPLOAD_CONFIG_FILENAME).write_text(json.dumps(UPLOAD_OK), encoding="utf-8") + monkeypatch.setattr(config_shapes, "UPLOAD", _add_required(config_shapes.UPLOAD)) + with pytest.raises(ValidationError, match="added is missing"): + upload.read_config(tmp_path) + + monkeypatch.undo() + (tmp_path / config.UPLOAD_CONFIG_FILENAME).write_text( + json.dumps(_with(UPLOAD_OK, max_bytes=_DROP)), encoding="utf-8" + ) + with pytest.raises(ValidationError): + upload.read_config(tmp_path) + monkeypatch.setattr(config_shapes, "UPLOAD", _not_required(config_shapes.UPLOAD, "max_bytes", 99)) + assert upload.read_config(tmp_path).max_bytes == 99 + + +def test_tasks_follows_a_field_made_required(tmp_path, monkeypatch): + (tmp_path / config.TASKS_CONFIG_FILENAME).write_text(json.dumps(TASKS_OK), encoding="utf-8") + assert tasks_config.read_config(tmp_path) is not None + monkeypatch.setattr(config_shapes, "TASKS", _add_required(config_shapes.TASKS)) + with pytest.raises(ValidationError, match="added is missing"): + tasks_config.read_config(tmp_path) + + +def test_remotes_follows_a_field_made_required(tmp_path, monkeypatch): + monkeypatch.setattr(config, "ROOT", tmp_path) + (tmp_path / config.PUBLISH_REMOTES_FILENAME).write_text( + json.dumps({"allowed_push_urls": ["u"]}), encoding="utf-8" + ) + assert git_publish.read_allowed_push_urls() == ["u"] + monkeypatch.setattr(config_shapes, "REMOTES", _add_required(config_shapes.REMOTES)) + with pytest.raises(typer.Exit): + git_publish.read_allowed_push_urls() + + +def test_caldav_follows_a_field_made_required(monkeypatch): + data = _with(CALDAV_OK, exclude_lists=_DROP) + assert caldav.CalDAVConfig.from_dict(data).exclude_lists == () + fields = tuple( + dataclasses.replace(f, required=True) if f.name == "exclude_lists" else f + for f in config_shapes.CALDAV.fields + ) + monkeypatch.setattr(config_shapes, "CALDAV", dataclasses.replace(config_shapes.CALDAV, fields=fields)) + with pytest.raises(ValidationError, match="exclude_lists is missing"): + caldav.CalDAVConfig.from_dict(data) + + +def test_superproductivity_follows_a_field_added_to_a_variant(monkeypatch): + data = {"access": "api", "api_token": "t", "timeout": "5"} + with pytest.raises(ValidationError, match="does not take"): + superproductivity.SuperProductivityConfig.from_dict(data) + shape = config_shapes.SUPERPRODUCTIVITY + api = dict(shape.variants)["api"] + (config_shapes.Field("timeout", "string", "x"),) + variants = (("api", api),) + tuple(v for v in shape.variants if v[0] != "api") + monkeypatch.setattr(config_shapes, "SUPERPRODUCTIVITY", dataclasses.replace(shape, variants=variants)) + assert superproductivity.SuperProductivityConfig.from_dict(data).api_token == "t" + + +def test_telemetry_follows_a_condition_in_the_record(tmp_path, monkeypatch): + (tmp_path / config.TELEMETRY_FILENAME).write_text(json.dumps({"keep_sessions": 500}), encoding="utf-8") + policy.reset_cache() + assert policy.resolve(tmp_path).keep_sessions == 500 + shape = config_shapes.TELEMETRY + fields = tuple( + dataclasses.replace(f, check=lambda v: None if v < 100 else "too many") if f.name == "keep_sessions" else f + for f in shape.fields + ) + monkeypatch.setattr(config_shapes, "TELEMETRY", dataclasses.replace(shape, fields=fields)) + policy.reset_cache() + assert policy.resolve(tmp_path).keep_sessions == policy.DEFAULT_KEEP_SESSIONS + policy.reset_cache() + + +# --- deviations: what doctor reports for telemetry ------------------------- + + +@pytest.mark.parametrize("data,field", [ + ({"enabled": "yes"}, "enabled"), + ({"enabled": 1}, "enabled"), + ({"max_session_bytes": 0}, "max_session_bytes"), + ({"keep_sessions": "250"}, "keep_sessions"), + ({"keep_sessions": 2.5}, "keep_sessions"), + ({"enabled": True, "verbose": True}, "verbose"), +]) +def test_a_telemetry_deviation_names_its_field(data, field): + found = config_shapes.deviations(config_shapes.TELEMETRY, data) + assert found and all(field in entry for entry in found) + + +def test_a_conforming_telemetry_file_has_no_deviation(): + for example in config_shapes.TELEMETRY.examples: + assert config_shapes.deviations(config_shapes.TELEMETRY, example) == [] + assert config_shapes.deviations(config_shapes.TELEMETRY, {"schema": 1, "enabled": False}) == [] + + +def test_every_example_passes_its_own_record(): + for shape in config_shapes.SHAPES: + for example in shape.examples: + assert config_shapes.check(shape, example).problems == [], shape.key + for section in config_shapes.TASKS.sections: + for example in section.examples: + assert config_shapes.check(section, example).problems == [], section.key + + +# --- the generated regions -------------------------------------------------- + + +def _page() -> str: + return (config.ROOT / reference_doc.CONFIGURATION).read_text(encoding="utf-8") + + +def _all_fields(shape, prefix=""): + for spec in shape.fields + tuple(f for _, fs in shape.variants for f in fs): + yield prefix + spec.name + yield from (prefix + spec.name + "." + sub.name for sub in spec.fields) + + +def test_the_page_carries_one_region_per_file_and_every_field_in_it(): + page = _page() + for name, attr in reference_doc.CONFIG_REGIONS: + shape = getattr(config_shapes, attr) + region = blocks.find(page, name) + assert region is not None, name + for path in _all_fields(shape): + assert f"`{path}`" in region, (name, path) + for section in shape.sections: + for path in _all_fields(section): + assert f"`{path}`" in region, (name, path) + + +@pytest.mark.parametrize("relative", [reference_doc.CONFIGURATION, "docs/how-to/set-up-the-mcp-server.md"]) +def test_no_json_example_for_these_files_stands_outside_a_region(relative): + text = blocks.mask_regions((config.ROOT / relative).read_text(encoding="utf-8")) + keys = {f.name for shape in config_shapes.SHAPES for f in shape.fields if f.name != "schema"} + keys |= {f.name for s in config_shapes.TASKS.sections for f in s.fields} + keys |= {f.name for s in config_shapes.TASKS.sections for _, fs in s.variants for f in fs} + for block in re.findall(r"```json\n(.*?)```", text, re.S): + assert not any(f'"{key}"' in block for key in keys), block + + +def _only_config_regions(monkeypatch): + real = reference_doc.regions + monkeypatch.setattr( + reference_doc, "regions", + lambda *a, **k: [r for r in real(*a, **k) if r.path == reference_doc.CONFIGURATION], + ) + + +def _hand_written(text: str) -> str: + for name, _ in reference_doc.CONFIG_REGIONS: + text = re.sub( + re.escape(blocks.open_marker(name)) + ".*?" + re.escape(blocks.close_marker(name)), + blocks.open_marker(name), text, flags=re.S, + ) + return text + + +def test_apply_fills_every_region_of_the_page_and_a_second_run_changes_nothing(tmp_path, monkeypatch): + """Four regions in one file: each is refreshed into the text the one + before it left, or only the last would survive.""" + target = tmp_path / reference_doc.CONFIGURATION + target.parent.mkdir(parents=True) + real = _page() + emptied = real + for name, _ in reference_doc.CONFIG_REGIONS: + emptied = blocks.replace(emptied, name, blocks.open_marker(name) + "\n" + blocks.close_marker(name) + "\n") + target.write_text(emptied, encoding="utf-8") + monkeypatch.setattr(config, "ROOT", tmp_path) + _only_config_regions(monkeypatch) + + result = runner.invoke(app, ["docs", "reference", "--apply"]) + assert result.exit_code == 0, result.output + applied = target.read_text(encoding="utf-8") + assert applied == real + assert _hand_written(applied) == _hand_written(emptied) + + result = runner.invoke(app, ["docs", "reference", "--apply"]) + assert "already current" in result.output + assert target.read_text(encoding="utf-8") == applied + + +def test_a_changed_record_without_apply_fails_docs_verify(monkeypatch): + assert not [i for i in docs_verify.check_reference_regions() if "configuration.md" in i] + monkeypatch.setattr(config_shapes, "REMOTES", _add_required(config_shapes.REMOTES)) + issues = [i for i in docs_verify.check_reference_regions() if reference_doc.CONFIGURATION in i] + assert len(issues) == 1 + assert "reference-config-remotes" in issues[0] and "docs reference --apply" in issues[0] + + +def test_the_configuration_regions_are_checked_in_an_instance_too(): + regions = [r for r in reference_doc.regions() if r.path == reference_doc.CONFIGURATION] + assert [r.name for r in regions] == [name for name, _ in reference_doc.CONFIG_REGIONS] + assert not any(r.origin_only for r in regions) diff --git a/tools/chemenu/tests/test_doctor.py b/tools/chemenu/tests/test_doctor.py index 97e3130..39d2d1b 100644 --- a/tools/chemenu/tests/test_doctor.py +++ b/tools/chemenu/tests/test_doctor.py @@ -521,6 +521,49 @@ def test_telemetry_never_fails_either_way(instance): assert _status(doctor.run_doctor(), "telemetry") == "OK" +def _telemetry_with(text: "str | None"): + """`check_telemetry` against a `.wikitool-telemetry.json` holding `text` + (absent for None), plus what the policy itself makes of it.""" + from chemenu.telemetry import policy + + path = config.ROOT / config.TELEMETRY_FILENAME + if text is None: + path.unlink(missing_ok=True) + else: + path.write_text(text, encoding="utf-8") + policy.reset_cache() + try: + return doctor.check_telemetry(), policy.resolve(config.ROOT) + finally: + policy.reset_cache() + + +@pytest.mark.parametrize("text", [None, '{"enabled": false}', '{"schema": 1, "enabled": true, ' + '"max_session_bytes": 1000, "keep_sessions": 9}']) +def test_telemetry_stays_ok_for_an_absent_or_conforming_file(instance, text): + check, _ = _telemetry_with(text) + assert check.status == "OK" + assert check.fix is None + + +@pytest.mark.parametrize("text,named,policy_reads", [ + ("{not json", "unreadable", (True, 5 * 1024 * 1024, 250)), + ("[true]", "not a JSON object", (True, 5 * 1024 * 1024, 250)), + ('{"enabled": "no"}', "enabled", (True, 5 * 1024 * 1024, 250)), + ('{"max_session_bytes": 0}', "max_session_bytes", (True, 5 * 1024 * 1024, 250)), + ('{"keep_sessions": "12"}', "keep_sessions", (True, 5 * 1024 * 1024, 12)), + ('{"enabled": false, "keep_session": 12}', "keep_session", (False, 5 * 1024 * 1024, 250)), +]) +def test_telemetry_warns_on_what_the_policy_reads_differently(instance, text, named, policy_reads): + """The policy stays silent and reads the file exactly as before; `doctor` + is where the deviation becomes visible, by field.""" + check, pol = _telemetry_with(text) + assert check.status == "WARN" + assert named in check.detail + assert config.TELEMETRY_FILENAME in check.fix + assert (pol.enabled, pol.max_session_bytes, pol.keep_sessions) == policy_reads + + def test_telemetry_reports_the_current_session_count_and_byte_total(instance, monkeypatch): from chemenu.telemetry.writer import TRACE_FILE, trace_root diff --git a/tools/chemenu/tests/test_reference_doc.py b/tools/chemenu/tests/test_reference_doc.py index 9eed31b..8dedbef 100644 --- a/tools/chemenu/tests/test_reference_doc.py +++ b/tools/chemenu/tests/test_reference_doc.py @@ -266,6 +266,7 @@ def _page_type(description: str = "A fixture type.") -> reference_doc.PageType: @pytest.fixture def tree(tmp_path, monkeypatch): + real_root = config.ROOT monkeypatch.setattr(config, "ROOT", tmp_path) records = {"frobnicate": _record("frobnicate"), "gate keep": _record("gate keep", ("keep",))} monkeypatch.setattr(cli_contract, "all_records", lambda: dict(records)) @@ -283,6 +284,13 @@ def tree(tmp_path, monkeypatch): path.write_text( f"{head}\n{blocks.open_marker(name)}\n{blocks.close_marker(name)}\n", encoding="utf-8" ) + # The configuration regions render from the real records, not from the + # fixture's; the current page keeps them out of what these tests count. + # test_config_shapes.py covers that page. + configuration = tmp_path / reference_doc.CONFIGURATION + configuration.write_text( + (real_root / reference_doc.CONFIGURATION).read_text(encoding="utf-8"), encoding="utf-8" + ) return tmp_path, records diff --git a/tools/chemenu/upload.py b/tools/chemenu/upload.py index 64b1b92..8f22ec7 100644 --- a/tools/chemenu/upload.py +++ b/tools/chemenu/upload.py @@ -49,10 +49,10 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, Optional -from chemenu import config +from chemenu import config, config_shapes from chemenu.errors import ValidationError -DEFAULT_IDENTITY_HEADER = "X-Forwarded-User" +DEFAULT_IDENTITY_HEADER = config_shapes.UPLOAD.field("identity_header").default # Submissions older than this stop counting toward a submitter's quota. A # rolling window rather than a calendar day - "resets at midnight" is a @@ -99,46 +99,20 @@ def read_config(root: "Path | str") -> Optional[UploadConfig]: "upload tool is offered at all, so a broken file is not treated as 'no limits' - " "fix it or delete it deliberately." ) from exc - if not isinstance(data, dict): - raise ValidationError(f"{config.UPLOAD_CONFIG_FILENAME} must contain a JSON object.") - - expected = ( - '{"schema": 1, "identity_header": "X-Forwarded-User", "max_bytes": 10485760, ' - '"allowed_extensions": [".md", ".pdf"], ' - '"quota": {"submissions_per_day": 20, "bytes_per_day": 52428800}}' - ) - try: - identity_header = str(data.get("identity_header") or DEFAULT_IDENTITY_HEADER) - max_bytes = int(data["max_bytes"]) - raw_extensions = data["allowed_extensions"] - extensions = tuple(sorted({str(ext).lower() for ext in raw_extensions})) - quota = data["quota"] - submissions_per_day = int(quota["submissions_per_day"]) - bytes_per_day = int(quota["bytes_per_day"]) - except (KeyError, TypeError, ValueError) as exc: + shape = config_shapes.UPLOAD + checked = config_shapes.check(shape, data) + if checked.problems: raise ValidationError( - f"{config.UPLOAD_CONFIG_FILENAME} is missing or misshapes a required field ({exc}). " - f"Expected: {expected}" - ) from exc - - if max_bytes <= 0: - raise ValidationError(f"{config.UPLOAD_CONFIG_FILENAME}: max_bytes must be positive.") - if submissions_per_day <= 0 or bytes_per_day <= 0: - raise ValidationError(f"{config.UPLOAD_CONFIG_FILENAME}: quota values must be positive.") - if not extensions: - raise ValidationError(f"{config.UPLOAD_CONFIG_FILENAME}: allowed_extensions must not be empty.") - not_dotted = [ext for ext in extensions if not ext.startswith(".")] - if not_dotted: - raise ValidationError( - f"{config.UPLOAD_CONFIG_FILENAME}: allowed_extensions must each start with '.': {not_dotted}" + f"{config.UPLOAD_CONFIG_FILENAME}: {checked.message()}. " + f"Expected: {config_shapes.render_expected(shape)}" ) - + values = checked.values return UploadConfig( - identity_header=identity_header, - max_bytes=max_bytes, - allowed_extensions=extensions, - submissions_per_day=submissions_per_day, - bytes_per_day=bytes_per_day, + identity_header=values["identity_header"], + max_bytes=values["max_bytes"], + allowed_extensions=values["allowed_extensions"], + submissions_per_day=values["quota"]["submissions_per_day"], + bytes_per_day=values["quota"]["bytes_per_day"], )