From 070cae6aad0ae7cb9931cfcd9080f80fcaf397b7 Mon Sep 17 00:00:00 2001 From: Torben Nehmer Date: Tue, 6 Oct 2026 22:10:09 +0200 Subject: [PATCH] feat: docs reference generates a skill catalogue and a page-type reference under docs/reference/ from the shipped defaults, checked and written in the origin repo only (#189) Files changed: - CHANGES.md - README.md - VERSION - docs/reference/README.md - docs/reference/commands/README.md - docs/reference/commands/types-instructions-and-docs.md - docs/reference/page-types.md - docs/reference/skills.md - instructions/dev/doc-pull-through.md - tools/CONTRACT.md - tools/README.md - tools/chemenu/commands/docs_verify.py - tools/chemenu/reference_doc.py - tools/chemenu/tests/test_reference_doc.py Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2 --- CHANGES.md | 21 +- README.md | 30 ++- VERSION | 2 +- docs/reference/README.md | 9 +- docs/reference/commands/README.md | 2 +- .../commands/types-instructions-and-docs.md | 11 +- docs/reference/page-types.md | 138 +++++++++++++ docs/reference/skills.md | 19 ++ instructions/dev/doc-pull-through.md | 4 +- tools/CONTRACT.md | 17 +- tools/README.md | 2 +- tools/chemenu/commands/docs_verify.py | 55 ++++-- tools/chemenu/reference_doc.py | 185 +++++++++++++++++- tools/chemenu/tests/test_reference_doc.py | 176 ++++++++++++++++- 14 files changed, 606 insertions(+), 65 deletions(-) create mode 100644 docs/reference/page-types.md create mode 100644 docs/reference/skills.md diff --git a/CHANGES.md b/CHANGES.md index 1904f74..fe2d6e4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,7 +14,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. --- -## 9.0.0-beta.4 - 2026-10-06 - tools/README.md module tree names reference_doc.py and the fourth view cli_contract renders +## 9.0.0-beta.5 - 2026-10-06 - docs reference: skill catalogue and page types under docs/reference/, generated from the shipped defaults in the origin repo only **Author:** Torben Nehmer @@ -29,6 +29,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. **Medium impact** - Changelog kompakt: ein Absatz pro Thema, Größenbudget, Migrationsverweis - docs reference: human command reference under docs/reference/, generated from the command records +- docs reference: skill catalogue and page types under docs/reference/, generated from the shipped defaults in the origin repo only **Low impact** - version bump: Abschlussmeldung nennt Stichpunkt oder Themenabsatz statt pauschal Prosa @@ -57,16 +58,16 @@ Migrationsdokument auf die Basis des Kandidaten, schreibt jeder `version bump` d grenzüberschreitenden Eintrag, der sein Dokument nicht nennt. Die Formregeln stehen nur noch in `instructions/dev/version-parts.md`. -### Kommandoreferenz für Menschen unter `docs/reference/` (#188) +### Generierte Referenz für Menschen unter `docs/reference/` (#188, #189) -Der neue Befehl `wikitool docs reference [--apply]` erzeugt aus den Kommando-Datensätzen eine Seite -je Gruppe unter `docs/reference/commands/`, eine Übersicht in deren `README.md` und -`docs/reference/exit-codes.md` mit den Gates und allen Ursachen für Exit 42. Die Seiten zeigen -Synopsis, Optionen, Beispiele, Verhalten, Eigenschaften in Worten und Exit-Codes mit Ursache, -nicht aber die an Agenten gerichteten ON-FAILURE-Reaktionen und NEVER-Zeilen. Titel und Kopfsatz -schreibt ein Mensch; das Werkzeug legt keine Seite an. `docs verify` meldet eine veraltete Region, -eine Gruppe ohne Seite und eine Seite ohne Gruppe, solange es `docs/` gibt. Das Inhaltsverzeichnis -kommt nie mehr in eine andere generierte Region hinein. +`wikitool docs reference [--apply]` erzeugt aus den Kommando-Datensätzen eine Seite je Gruppe unter +`docs/reference/commands/` samt Übersicht und `exit-codes.md` mit Gates und Exit-42-Ursachen, ohne +die an Agenten gerichteten Reaktionen und NEVER-Zeilen. Dazu kommen `skills.md`, der Katalog der +ausgelieferten Skills aus ihrer `description`, und `page-types.md` mit Subtypen, Zielverzeichnissen +und Pflichtfeldern je `root: kb`-Type-Spec. Diese beiden beschreiben die ausgelieferten Vorgaben +und werden nur im Ursprungs-Repo erzeugt und geprüft; in einer Instanz lässt `docs reference` sie +unberührt. Titel und Kopfsatz schreibt ein Mensch. `docs verify` meldet veraltete Regionen und +fehlende Seiten, solange es `docs/` gibt. --- diff --git a/README.md b/README.md index 7dc936e..16dc17f 100644 --- a/README.md +++ b/README.md @@ -314,11 +314,13 @@ way `task new` opened it, or the way the weekly review proposes it for a Why the split runs this way, rather than syncing the two: [docs/explanation/knowledge-and-commitment.md](docs/explanation/knowledge-and-commitment.md). -## Entity Types +## Page Types -Entities are subtyped as codebase, system, tool, technology, or person, and each subtype has -its own directory under `kb/entities/`. The authoritative list - and where each one is -written - is declared by the type-spec, so ask the tool rather than a table here: +Every page type the stack ships - entity, concept, source, comparison, project - with its +subtypes, the directory each is written to and its required fields, is listed in +[docs/reference/page-types.md](docs/reference/page-types.md), generated from the type-specs. An +instance adopts those type-specs and may change them, so for the types *this* instance has, ask +the tool: ```bash tools/wikitool types list @@ -340,16 +342,9 @@ tools/wikitool types describe entity `AGENTS.md` is the cross-cutting schema/policy; the step-by-step procedures themselves live as independently-discoverable skills under `.agents/skills/` -(mirrored to `.claude/skills/` for Claude Code via `tools/wikitool instructions sync`): - -| Skill | Purpose | -|-------|---------| -| `wiki-ingest` | Process a new source into the wiki: read it, discuss its content and any commitment with the user, promote it from `incoming/` into `raw/`, then source summary, entity/concept pages, cross-references, index/log, publish | -| `wiki-query` | Answer a question from the compiled wiki; read-only, can optionally file a valuable answer back as a new page | -| `wiki-lint` | Health-check the wiki: structural scan, raw coverage, semantic review | -| `wiki-manage` | Create a new entity/concept/source/comparison page, or update an existing page with new information | -| `wiki-status` | Read-only snapshot: page counts, orphans, uncovered raw files, most-connected pages | -| `gtd-weekly-review` | Turns `wikitool review`'s findings into decisions and page updates - the GTD weekly review | +(mirrored to `.claude/skills/` for Claude Code via `tools/wikitool instructions sync`). +Which skills ship, what each does and which requests start it: +[docs/reference/skills.md](docs/reference/skills.md), generated from the skills themselves. Each skill's underlying mechanical work (frontmatter, cross-references, index/log, decay math, publishing) is delegated to `tools/wikitool` - never hand-edited. @@ -580,11 +575,8 @@ The LLM will create and maintain: - `kb/index.md` - Always up-to-date catalog - `kb/log.md` - Complete audit trail - `kb/provenance.md` - Raw-file reverse index -- Source pages in `kb/sources/` -- Entity pages in `kb/entities/` -- Concept pages in `kb/concepts/` -- Comparison pages in `kb/comparisons/` -- Project (Vorhaben) pages in `kb/gtd/` +- Pages of every page type, each in the directory + [docs/reference/page-types.md](docs/reference/page-types.md) names for it - Lint reports, session traces and eval scores in `reports/` (gitignored) ## Changelog diff --git a/VERSION b/VERSION index 696cc4d..34e2bcb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -9.0.0-beta.4 +9.0.0-beta.5 diff --git a/docs/reference/README.md b/docs/reference/README.md index d22a791..6914c9d 100644 --- a/docs/reference/README.md +++ b/docs/reference/README.md @@ -9,6 +9,11 @@ written by hand. generated from the same records as `tools/wikitool -h`. - [Exit codes and gates](exit-codes.md) - which commands can stop and wait for a person, and why. +- [Skill catalogue](skills.md) - the skills the stack ships, what each does and which requests + start it, generated from their `SKILL.md`. +- [Page types](page-types.md) - every page type the stack ships for `kb/`, its subtypes, where + its pages are written and its required fields, generated from the type-specs. -Page types are not generated here yet; until they are, a page type is described by -`tools/wikitool types describe `. +The last two describe the stack's shipped defaults, which an instance may change or extend; +`tools/wikitool instructions list` and `tools/wikitool types describe ` show what one +instance has. diff --git a/docs/reference/commands/README.md b/docs/reference/commands/README.md index 04024b7..6ce128c 100644 --- a/docs/reference/commands/README.md +++ b/docs/reference/commands/README.md @@ -123,7 +123,7 @@ Full records: [types-instructions-and-docs.md](types-instructions-and-docs.md) - [`docs toc`](types-instructions-and-docs.md#docs-toc) - Create, refresh or remove the generated table-of-contents region. - [`docs prerequisites`](types-instructions-and-docs.md#docs-prerequisites) - Regenerate `INSTALL.md`'s prerequisites lists from `tools/prerequisites.txt`. - [`docs contract`](types-instructions-and-docs.md#docs-contract) - Regenerate `tools/CONTRACT.md`'s `` region. -- [`docs reference`](types-instructions-and-docs.md#docs-reference) - Regenerate the human command reference under `docs/reference/`. +- [`docs reference`](types-instructions-and-docs.md#docs-reference) - Regenerate the human reference under `docs/reference/`. ## Telemetry diff --git a/docs/reference/commands/types-instructions-and-docs.md b/docs/reference/commands/types-instructions-and-docs.md index 75fefd5..56dc657 100644 --- a/docs/reference/commands/types-instructions-and-docs.md +++ b/docs/reference/commands/types-instructions-and-docs.md @@ -288,7 +288,7 @@ tools/wikitool docs verify - Every reference file `docs toc` covers carries the current table-of-contents region for its own headings - missing and stale are one check. - Every relative markdown link in one of those reference files resolves to an existing file. A target's `#anchor` suffix is stripped first, and code fences and inline code spans are masked before scanning, so link syntax shown as an example is not mistaken for a real reference. - Every file under `docs/` sits in one of its areas - `tutorials/`, `how-to/`, `reference/`, `explanation/` or `dev/` - never directly in `docs/` or in another directory. -- If `docs/` exists: every generated region of the human command reference under `docs/reference/` matches what `docs reference` would write, every `cli_contract.GROUPS` entry has its page under `docs/reference/commands/`, and no page there belongs to no group. +- If `docs/` exists: every generated region of the human reference under `docs/reference/` matches what `docs reference` would write, every `cli_contract.GROUPS` entry has its page under `docs/reference/commands/`, and no page there belongs to no group. The skill catalogue and the page types are checked only in the origin repo (no `.wikitool-release.json`): they describe the stack's shipped defaults, which an instance's own type-specs and instructions may differ from. - `INSTALL.md` carries one generated `` region per platform value of `tools/prerequisites.txt` (`prerequisites-` for a platform-specific one), each current; and the `` markers in `instructions/setup-instance.md` and `INSTALL.md` name the same set of keys, so a question the agent asks is never one the human guide leaves out, nor the reverse. - `VERSION` parses and the newest versioned `CHANGES.md` entry names it. An entry that crosses the compatibility boundary from the last release carries `**Breaking Change:**`, has a migration document or `**Migration:** none required`, and its `**Migration:**` line names every required migration document targeting it. - The newest versioned `CHANGES.md` entry is at most 32 000 bytes (UTF-8) as `version notes` prints it, and each of its `###` sections is exactly one paragraph of at most 1 000 bytes, heading line excluded. Older entries and a changelog with no versioned entry are not checked. @@ -312,8 +312,8 @@ tools/wikitool docs verify - `1` - A type-spec's own frontmatter fails its schema - `1` - A subtype template `types/..md` has no type-spec with a `subtype_field:` beside it, names a value outside that field's enum, or carries frontmatter - `1` - A shipped `.md`/`.template` cites an issue number -- `1` - A region of the human command reference under `docs/reference/` is stale -- `1` - A page of the human command reference is missing or lacks its marker pair, or a page under `docs/reference/commands/` belongs to no group +- `1` - A region of the human reference under `docs/reference/` is stale +- `1` - A page of the human reference is missing or lacks its marker pair, or a page under `docs/reference/commands/` belongs to no group - `1` - A reference file's table-of-contents region is missing or stale - `1` - A file sits directly in `docs/`, or in a directory there that is not one of its five areas - `1` - A reference file's relative markdown link does not resolve to an existing file @@ -325,7 +325,7 @@ tools/wikitool docs verify - `wikitool docs toc` - regenerates tables of contents - `wikitool docs contract` - regenerates the commands region -- `wikitool docs reference` - regenerates the human command reference +- `wikitool docs reference` - regenerates the human reference - `wikitool docs prerequisites` - regenerates `INSTALL.md`'s prerequisites lists - `wikitool instructions verify` - the same kind of check for `instructions/` @@ -471,7 +471,7 @@ tools/wikitool docs contract --apply ## `docs reference` -Regenerate the human command reference under `docs/reference/`. +Regenerate the human reference under `docs/reference/`. **Synopsis** @@ -494,6 +494,7 @@ tools/wikitool docs reference --apply **Behaviour** - Rewrites the generated region of every page of the human command reference from the `cli_contract` records: one `docs/reference/commands/.md` per `cli_contract.GROUPS` entry (``), the overview in `docs/reference/commands/README.md` (``), and the gates and exit-42 causes in `docs/reference/exit-codes.md` (``). +- Also, in the origin repo only, the skill catalogue `docs/reference/skills.md` (``: one row per shipped `instructions//SKILL.md`, its description split at the sentence starting "Use when") and the page types `docs/reference/page-types.md` (``: one section per `root: kb` type-spec, from what `types describe` reads). Both describe the stack's shipped defaults. In an instance - a tree with `.wikitool-release.json` - they are left alone and the output says so: an instance's adopted type-specs and own instructions may differ, and rewriting a shipped page there would block the next `dist upgrade`. - Renders each command's summary, synopsis, options, examples, notes, properties in words, exit codes with their causes and see-also - but not the ON FAILURE reactions or NEVER, which address an agent; `tools/CONTRACT.md` keeps those. - Refreshes the table of contents of each page it writes in the same step. - Never creates a page or places a region: a page's title and one-sentence head are written by hand. A group's page is named after the group - lower case, every run of other characters one hyphen. diff --git a/docs/reference/page-types.md b/docs/reference/page-types.md new file mode 100644 index 0000000..f0ff30a --- /dev/null +++ b/docs/reference/page-types.md @@ -0,0 +1,138 @@ +# Page types + +Every page type the stack ships for `kb/`: what it is for, where its pages are written, its +subtypes and its required frontmatter fields, generated from the type-specs under `types/` and +their schemas. + +This page describes the stack's shipped defaults. An instance adopts these type-specs from their +`.template` and may change them, or add types of its own; `tools/wikitool types list` and +`tools/wikitool types describe ` show what this instance has. + + +## Contents + +- [`comparison`](#comparison) +- [`concept`](#concept) +- [`entity`](#entity) +- [`project`](#project) +- [`source`](#source) + + + +## `comparison` + +Structured type for comparison pages that set several entities or approaches against one another + +- Written to: `kb/comparisons/` + +**Required fields** + +- `type` - string +- `created` - string +- `entities` - array +- `summary` - string + +## `concept` + +Base type for concept pages - architectures, patterns, protocols, workflows, decisions or problems + +- Written to: `kb/concepts//` + +**Subtypes** (field `concept_type`) + +| Value | Written to | Index section | +|---|---|---| +| `architecture` | `kb/concepts/architectures/` | Architekturen | +| `pattern` | `kb/concepts/patterns/` | Muster | +| `protocol` | `kb/concepts/protocols/` | Protokolle | +| `workflow` | `kb/concepts/workflows/` | Abläufe | +| `decision` | `kb/concepts/decisions/` | Entscheidungen | +| `problem` | `kb/concepts/problems/` | Problemstellungen | + +**Required fields** + +- `type` - string +- `concept_type` - string, one of `architecture`, `pattern`, `protocol`, `workflow`, `decision`, `problem` +- `created` - string +- `modified` - string +- `provenance` - string, one of `sourced`, `general`, `mixed` +- `summary` - string + +## `entity` + +Base type for entity pages - codebases, systems, tools, technologies, people or organizations + +- Written to: `kb/entities//` + +**Subtypes** (field `entity_type`) + +| Value | Written to | Index section | +|---|---|---| +| `codebase` | `kb/entities/codebases/` | Codebasen | +| `system` | `kb/entities/systems/` | Systeme | +| `tool` | `kb/entities/tools/` | Werkzeuge | +| `technology` | `kb/entities/technologies/` | Technologien | +| `person` | `kb/entities/people/` | Personen | +| `organization` | `kb/entities/organizations/` | Organisationen | + +**Required fields** + +- `type` - string +- `entity_type` - string, one of `codebase`, `system`, `tool`, `technology`, `person`, `organization` +- `created` - string +- `modified` - string +- `provenance` - string, one of `sourced`, `general`, `mixed` +- `summary` - string + +## `project` + +Structured type for a committed initiative (a GTD project) - goal, participants, durable status, open loops + +- Written to: `kb/gtd//` + +**Subtypes** (field `responsibility`) + +| Value | Written to | Index section | +|---|---|---| +| `haus` | `kb/gtd/haus/` | Haus | +| `finanzen` | `kb/gtd/finanzen/` | Finanzen | +| `technik` | `kb/gtd/technik/` | Technik | + +**Required fields** + +- `type` - string +- `state` - string, one of `active`, `dormant`, `completed`, `abandoned` +- `responsibility` - string, one of `haus`, `finanzen`, `technik` +- `created` - string +- `modified` - string +- `provenance` - string, one of `sourced`, `general`, `mixed` +- `summary` - string + +## `source` + +Structured type for source pages that record and summarize ingested raw material + +- Written to: `kb/sources//` +- Title prefix: `Source - ` + +**Subtypes** (field `source_type`) + +| Value | Written to | Index section | +|---|---|---| +| `transcript` | `kb/sources/transcripts/` | Transkripte | +| `analysis` | `kb/sources/analyses/` | Analysen | +| `article` | `kb/sources/articles/` | Artikel | +| `document` | `kb/sources/documents/` | Dokumente | +| `notes` | `kb/sources/notes/` | Notizen | +| `tracker` | `kb/sources/trackers/` | Tracker-Exporte | +| `unclassified` | `kb/sources/unclassified/` | Unklassifiziert | + +**Required fields** + +- `type` - string +- `source_type` - string, one of `transcript`, `analysis`, `article`, `document`, `notes`, `tracker`, `unclassified` +- `author` - string +- `raw_files` - array +- `date` - string +- `summary` - string + diff --git a/docs/reference/skills.md b/docs/reference/skills.md new file mode 100644 index 0000000..9b4b81d --- /dev/null +++ b/docs/reference/skills.md @@ -0,0 +1,19 @@ +# Skill catalogue + +The skills the stack ships, as an agent's harness sees them once `tools/wikitool instructions sync` +has published them: each one's name, what it does and the requests it answers, generated from its +`instructions//SKILL.md`. Any skill also runs when you ask for it by name. + +This page describes the stack's shipped defaults. An instance may add instructions of its own; +`tools/wikitool instructions list` shows what this instance has. + + +| Skill | What it does | Use it when | +|---|---|---| +| `gtd-weekly-review` | Turns the findings from `wikitool review` into decisions and page updates - the GTD Weekly Review, with a machine that prepares the list instead of a human reconstructing it from memory. | the user asks for "the weekly review", "review my projects", "what's stalled", or after `wikitool review` has findings nobody has acted on yet. | +| `wiki-ingest` | Processes a new source file into the LLM wiki - extracts entities and concepts, creates a source summary page, files a tracker item for any commitment the source also carries, cross-references, rebuilds indexes, and publishes. | the user drops a file or folder into incoming/ or raw/, names a URL to ingest, or says "ingest <file>", "ingest <url>", "process this source", "add this to the wiki" - or just "ingest" with nothing named, which takes the oldest entry waiting in incoming/ - or asks to "update the captured repositories" / "pull the repo docs", which takes the next bundle `raw status` reports as changed. | +| `wiki-lint` | Checks the health of the LLM wiki - broken links, orphan pages, uncovered raw files, stale claims, duplicated rules, missing cross-references. | the user says "lint the wiki", "health-check the wiki", or periodically every 10 sources per the Maintenance Schedule. | +| `wiki-manage` | Creates a new wiki page (entity, concept, source, comparison) or updates an existing page with new information, including cross-references, index/log, and publishing. | the user says "create a new entity/concept/comparison", "add a page for X", "update the X page", or new information needs integrating into an existing page. | +| `wiki-query` | Answers a question using the LLM wiki's compiled knowledge - read-only, cites sources, can file a valuable answer back as a new page. | the user asks a question about entities, projects, concepts, or anything the wiki might know, or says "query the wiki", "what do we know about X", "search the wiki". | +| `wiki-status` | Shows a quick read-only snapshot of the LLM wiki - page counts, orphan pages, uncovered raw files, recent activity. | the user says "wiki status", "show wiki statistics", "what's new", or wants a quick health snapshot without running a full lint. | + diff --git a/instructions/dev/doc-pull-through.md b/instructions/dev/doc-pull-through.md index d867084..a1d8762 100644 --- a/instructions/dev/doc-pull-through.md +++ b/instructions/dev/doc-pull-through.md @@ -39,11 +39,11 @@ touched; a row that does not apply needs no action. | A workflow, stage, or command a human operates by hand | Whichever of `README.md`, `EVALS.md`, `tools/README.md`, `INSTALL.md`, `INSTALL-MCP.md` or the pages under `docs/tutorials/`, `docs/how-to/` and `docs/reference/` names it - AGENTS.md § File naming says which document is for which reader. A new page goes into the area its kind of documentation belongs to; that area's `README.md` says which | | The reasoning behind a gate, boundary, or design decision | The `docs/explanation/` page that carries it, if one exists (AGENTS.md § File naming lists all six reached from AGENTS.md itself, plus a seventh reached only from CLAUDE.md). **A decision with no page yet is the gap worth closing**: reasoning that lives only in a Gitea issue never ships - `dist export` carries `docs/explanation/` and no issue tracker, so a distributed instance gets the mechanism without the why | | A task-tracker adapter (`tools/chemenu/tasks/`), its recorded fixtures, or the live suite | [instructions/dev/tracker-testing.md](tracker-testing.md), and `MANIFEST.json` beside the fixtures when they were re-recorded | - | A skill's own step sequence or catalogue | The skill's `SKILL.md` source under `instructions//` or `instructions/dev//` | + | A skill's own step sequence or catalogue | The skill's `SKILL.md` source under `instructions//` or `instructions/dev//`; for a shipped skill (not under `instructions/dev/`) whose name or `description` changed, or one added or removed, `wikitool docs reference --apply` for the skill catalogue `docs/reference/skills.md` | | A per-checkout configuration file an instance owns (`.wikitool-tasks.json`, `.wikitool-telemetry.json`, `.wikitool-remotes.json`, `.wikitool-upload.json`) | [INSTALL.md](../../INSTALL.md) § Konfiguration, where an operator looks the shape up; the [setup-instance.md](../setup-instance.md) decision point that offers it during setup; and `doctor`'s own row in [tools/CONTRACT.md](../../tools/CONTRACT.md), since `doctor` is what reports the file's state | | An installation instruction - [preflight.md](../preflight.md), [setup-instance.md](../setup-instance.md), [bootstrap.md](../bootstrap.md), [upgrade-instance.md](../upgrade-instance.md) - or `tools/prerequisites.txt` | [INSTALL.md](../../INSTALL.md), the human guide to the same procedure. Read it against the instruction: what to prepare, the sentence for the agent, what the agent asks, where it stops and why. `docs verify` checks only the two enumerable overlaps - the prerequisites lists, which `wikitool docs prerequisites --apply` regenerates from the manifest, and the setup questions: a question the agent asks the user carries `` where it is asked in `setup-instance.md`, and `INSTALL.md` § "Was der Agent dich fragt" names it with the same marker. Every other sentence is this session's to compare. `INSTALL.md` does not retell the steps, so a change to their order or wording alone moves nothing there | | [dev-setup.md](dev-setup.md), the release workflow (`version bump`/`version release`/`publish`/CI), or another developer-facing process | [docs/dev/development.md](../../docs/dev/development.md), read against it the same way - nothing checks this pair at all. A page under `docs/dev/` is never shipped, so it may link into `instructions/dev/` and cite issue numbers | - | A new page type the stack requires, or a new collection | Its type-spec and `COLLECTION.md` (both as the `.template` an instance adopts), the collection table in [kb/CONTRACT.md](../../kb/CONTRACT.md), and **both adoption paths**: [setup-instance.md](../setup-instance.md) for a fresh instance and [upgrade-instance.md](../upgrade-instance.md) for an existing one, where an unadopted template is what `docs verify` refuses | + | A new page type the stack requires, or a new collection | Its type-spec and `COLLECTION.md` (both as the `.template` an instance adopts), the collection table in [kb/CONTRACT.md](../../kb/CONTRACT.md), and **both adoption paths**: [setup-instance.md](../setup-instance.md) for a fresh instance and [upgrade-instance.md](../upgrade-instance.md) for an existing one, where an unadopted template is what `docs verify` refuses. A new or changed `root: kb` type-spec or its schema - description, `layout:`, subtype enum, required fields - also needs `wikitool docs reference --apply` for `docs/reference/page-types.md` | 3. **A heading you changed means a table of contents to regenerate - by the tool, never by hand.** Every reference file over 100 lines carries one (`AGENTS.md`, the stage contracts, diff --git a/tools/CONTRACT.md b/tools/CONTRACT.md index cf028b4..864cb34 100644 --- a/tools/CONTRACT.md +++ b/tools/CONTRACT.md @@ -145,7 +145,7 @@ docs verify read idempotent budget:counted exit:0,1 docs toc write idempotent budget:counted exit:0 Create, refresh or remove the generated table-of-contents region. docs prerequisites write idempotent budget:counted exit:0,1 Regenerate `INSTALL.md`'s prerequisites lists from `tools/prerequisites.txt`. docs contract write idempotent budget:counted exit:0,1 Regenerate `tools/CONTRACT.md`'s `` region. -docs reference write idempotent budget:counted exit:0,1 Regenerate the human command reference under `docs/reference/`. +docs reference write idempotent budget:counted exit:0,1 Regenerate the human reference under `docs/reference/`. eval sessions read idempotent budget:exempt exit:0 List the sessions that have a trace under `reports/telemetry/`. eval score read idempotent budget:exempt exit:0,1 Score one traced session. dist export write idempotent budget:counted exit:0,1 Write a contentless, distributable copy of this repo's machinery. @@ -2516,8 +2516,8 @@ Check the docs that mirror the code. - 1 A type-spec's own frontmatter fails its schema - 1 A subtype template `types/..md` has no type-spec with a `subtype_field:` beside it, names a value outside that field's enum, or carries frontmatter - 1 A shipped `.md`/`.template` cites an issue number -- 1 A region of the human command reference under `docs/reference/` is stale -- 1 A page of the human command reference is missing or lacks its marker pair, or a page under `docs/reference/commands/` belongs to no group +- 1 A region of the human reference under `docs/reference/` is stale +- 1 A page of the human reference is missing or lacks its marker pair, or a page under `docs/reference/commands/` belongs to no group - 1 A reference file's table-of-contents region is missing or stale - 1 A file sits directly in `docs/`, or in a directory there that is not one of its five areas - 1 A reference file's relative markdown link does not resolve to an existing file @@ -2534,8 +2534,8 @@ Check the docs that mirror the code. - A type-spec's own frontmatter fails its schema -> Fix the field, or add a matching line to `types/type-spec.schema.yaml` if the field is legitimately new - A subtype template `types/..md` has no type-spec with a `subtype_field:` beside it, names a value outside that field's enum, or carries frontmatter -> Rename the file to the type and value it was meant for, delete it, or remove its frontmatter block - A shipped `.md`/`.template` cites an issue number -> Say what was decided instead of pointing at where, or move the pointer behind a `` block -- A region of the human command reference under `docs/reference/` is stale -> Run `docs reference --apply`, then re-run -- A page of the human command reference is missing or lacks its marker pair, or a page under `docs/reference/commands/` belongs to no group -> Create the page with a title, a one-sentence head and the marker pair the message names (or add the pair), then run `docs reference --apply`; `git mv` or `git rm` a page left over from a renamed or removed group +- A region of the human reference under `docs/reference/` is stale -> Run `docs reference --apply`, then re-run +- A page of the human reference is missing or lacks its marker pair, or a page under `docs/reference/commands/` belongs to no group -> Create the page with a title, a one-sentence head and the marker pair the message names (or add the pair), then run `docs reference --apply`; `git mv` or `git rm` a page left over from a renamed or removed group - A reference file's table-of-contents region is missing or stale -> Run `docs toc --apply`, then re-run - A file sits directly in `docs/`, or in a directory there that is not one of its five areas -> Move it (`git mv`) into the area its kind of documentation belongs to, and fix the links that pointed at the old path - A reference file's relative markdown link does not resolve to an existing file -> Fix the `../` count or the target's name @@ -2559,7 +2559,7 @@ Check the docs that mirror the code. - Every reference file `docs toc` covers carries the current table-of-contents region for its own headings - missing and stale are one check. - Every relative markdown link in one of those reference files resolves to an existing file. A target's `#anchor` suffix is stripped first, and code fences and inline code spans are masked before scanning, so link syntax shown as an example is not mistaken for a real reference. - Every file under `docs/` sits in one of its areas - `tutorials/`, `how-to/`, `reference/`, `explanation/` or `dev/` - never directly in `docs/` or in another directory. -- If `docs/` exists: every generated region of the human command reference under `docs/reference/` matches what `docs reference` would write, every `cli_contract.GROUPS` entry has its page under `docs/reference/commands/`, and no page there belongs to no group. +- If `docs/` exists: every generated region of the human reference under `docs/reference/` matches what `docs reference` would write, every `cli_contract.GROUPS` entry has its page under `docs/reference/commands/`, and no page there belongs to no group. The skill catalogue and the page types are checked only in the origin repo (no `.wikitool-release.json`): they describe the stack's shipped defaults, which an instance's own type-specs and instructions may differ from. - `INSTALL.md` carries one generated `` region per platform value of `tools/prerequisites.txt` (`prerequisites-` for a platform-specific one), each current; and the `` markers in `instructions/setup-instance.md` and `INSTALL.md` name the same set of keys, so a question the agent asks is never one the human guide leaves out, nor the reverse. - `VERSION` parses and the newest versioned `CHANGES.md` entry names it. An entry that crosses the compatibility boundary from the last release carries `**Breaking Change:**`, has a migration document or `**Migration:** none required`, and its `**Migration:**` line names every required migration document targeting it. - The newest versioned `CHANGES.md` entry is at most 32 000 bytes (UTF-8) as `version notes` prints it, and each of its `###` sections is exactly one paragraph of at most 1 000 bytes, heading line excluded. Older entries and a changelog with no versioned entry are not checked. @@ -2569,7 +2569,7 @@ Check the docs that mirror the code. - `wikitool docs toc` - regenerates tables of contents - `wikitool docs contract` - regenerates the commands region -- `wikitool docs reference` - regenerates the human command reference +- `wikitool docs reference` - regenerates the human reference - `wikitool docs prerequisites` - regenerates `INSTALL.md`'s prerequisites lists - `wikitool instructions verify` - the same kind of check for `instructions/` @@ -2708,7 +2708,7 @@ Regenerate `tools/CONTRACT.md`'s `` region. #### `docs reference` -Regenerate the human command reference under `docs/reference/`. +Regenerate the human reference under `docs/reference/`. **SYNOPSIS** @@ -2743,6 +2743,7 @@ Regenerate the human command reference under `docs/reference/`. **NOTES** - Rewrites the generated region of every page of the human command reference from the `cli_contract` records: one `docs/reference/commands/.md` per `cli_contract.GROUPS` entry (``), the overview in `docs/reference/commands/README.md` (``), and the gates and exit-42 causes in `docs/reference/exit-codes.md` (``). +- Also, in the origin repo only, the skill catalogue `docs/reference/skills.md` (``: one row per shipped `instructions//SKILL.md`, its description split at the sentence starting "Use when") and the page types `docs/reference/page-types.md` (``: one section per `root: kb` type-spec, from what `types describe` reads). Both describe the stack's shipped defaults. In an instance - a tree with `.wikitool-release.json` - they are left alone and the output says so: an instance's adopted type-specs and own instructions may differ, and rewriting a shipped page there would block the next `dist upgrade`. - Renders each command's summary, synopsis, options, examples, notes, properties in words, exit codes with their causes and see-also - but not the ON FAILURE reactions or NEVER, which address an agent; `tools/CONTRACT.md` keeps those. - Refreshes the table of contents of each page it writes in the same step. - Never creates a page or places a region: a page's title and one-sentence head are written by hand. A group's page is named after the group - lower case, every run of other characters one hyphen. diff --git a/tools/README.md b/tools/README.md index 5acb365..dda3471 100644 --- a/tools/README.md +++ b/tools/README.md @@ -89,7 +89,7 @@ tools/ filelock.py an exclusive lock on an open file, flock on POSIX and msvcrt on Windows - the only module that imports either prerequisites.py prerequisites.txt read from Python, plus the platform and long-path questions `doctor` asks install_doc.py INSTALL.md held to the instructions: its prerequisites lists generated from prerequisites.txt, its setup questions matched to setup-instance.md's markers - reference_doc.py the human command reference under docs/reference/: one page per command group, an overview and the exit-42 causes, rendered from the records without what addresses an agent + reference_doc.py the human reference under docs/reference/: one page per command group, an overview and the exit-42 causes, rendered from the records without what addresses an agent; and, in the origin repo only, the skill catalogue and the page types, rendered from the shipped SKILL.md files and type-specs corpus_cache.py one parsed corpus per commit, never cached while the tree is dirty kb_scan.py page iteration/loading over kb/ blocks.py generated regions in a page body, found by marker rather than by heading diff --git a/tools/chemenu/commands/docs_verify.py b/tools/chemenu/commands/docs_verify.py index 9ba3d6d..f9fe380 100644 --- a/tools/chemenu/commands/docs_verify.py +++ b/tools/chemenu/commands/docs_verify.py @@ -337,19 +337,24 @@ def check_commands_region() -> list[str]: def check_reference_regions() -> list[str]: - """Every generated region of the human command reference under - `docs/reference/` matches what `docs reference` would write, every - `GROUPS` entry has its page, and no page is left over from a group that - is gone. + """Every generated region of the human reference under `docs/reference/` + matches what `docs reference` would write, every `GROUPS` entry has its + page, and no page is left over from a group that is gone. A tree without `docs/` passes, as for `check_docs_layout`: `docs verify` says what a present `docs/` must hold, not that one must exist. With `docs/` present, every page is required - a deleted one is drift too. + + In an instance the skill catalogue and the page types are not checked at + all: they describe the stack's shipped defaults, which an instance's own + adopted type-specs and instructions are free to differ from + (`reference_doc.applicable`). """ if not (config.ROOT / "docs").is_dir(): return [] issues = [] - for region in reference_doc.regions(): + checked, _ = reference_doc.applicable(reference_doc.regions()) + for region in checked: path = config.ROOT / region.path if not path.is_file(): issues.append( @@ -1288,10 +1293,12 @@ def check_breaking_change_for_boundary() -> list[str]: "Every file under `docs/` sits in one of its areas - `tutorials/`, `how-to/`, " "`reference/`, `explanation/` or `dev/` - never directly in `docs/` or in another " "directory.", - "If `docs/` exists: every generated region of the human command reference under " + "If `docs/` exists: every generated region of the human reference under " "`docs/reference/` matches what `docs reference` would write, every `cli_contract.GROUPS` " "entry has its page under `docs/reference/commands/`, and no page there belongs to no " - "group.", + "group. The skill catalogue and the page types are checked only in the origin repo " + "(no `.wikitool-release.json`): they describe the stack's shipped defaults, which an " + "instance's own type-specs and instructions may differ from.", "`INSTALL.md` carries one generated `` region per " "platform value of `tools/prerequisites.txt` (`prerequisites-` for a " "platform-specific one), each current; and the `` markers " @@ -1345,11 +1352,11 @@ def check_breaking_change_for_boundary() -> list[str]: "behind a `` block", ), cli_contract.Failure( - cause="A region of the human command reference under `docs/reference/` is stale", + cause="A region of the human reference under `docs/reference/` is stale", reaction="Run `docs reference --apply`, then re-run", ), cli_contract.Failure( - cause="A page of the human command reference is missing or lacks its marker pair, or " + cause="A page of the human reference is missing or lacks its marker pair, or " "a page under `docs/reference/commands/` belongs to no group", reaction="Create the page with a title, a one-sentence head and the marker pair the " "message names (or add the pair), then run `docs reference --apply`; `git mv` or " @@ -1397,7 +1404,7 @@ def check_breaking_change_for_boundary() -> list[str]: see_also=( "`wikitool docs toc` - regenerates tables of contents", "`wikitool docs contract` - regenerates the commands region", - "`wikitool docs reference` - regenerates the human command reference", + "`wikitool docs reference` - regenerates the human reference", "`wikitool docs prerequisites` - regenerates `INSTALL.md`'s prerequisites lists", "`wikitool instructions verify` - the same kind of check for `instructions/`", ), @@ -1434,6 +1441,10 @@ def verify(): from chemenu.type_resolver import resolver + reference_scope = ( + " (skill catalogue and page types are checked in the origin repo)" + if reference_doc.in_instance() else "" + ) success( f"Docs verified: {len(registered_commands())} command(s) with a cli_contract record, " f"{len(kb_collections.iter_kb_collections())} collection(s) and " @@ -1442,7 +1453,7 @@ def verify(): f"{len(IGNORE_CANARIES)} ignore canaries clear, " f"no issue references in {len(shipped_prose())} shipped document(s) or command help, " f"every docs/ file in one of its {len(DOCS_AREAS)} areas, " - f"the command reference under {reference_doc.REFERENCE_DIR}/ current, " + f"the reference under {reference_doc.REFERENCE_DIR}/ current{reference_scope}, " f"tables of contents current and every link resolving on " f"{len(toc.target_files())} reference file(s), " f"{install_doc.INSTALL_DOC} in step with tools/prerequisites.txt and " @@ -1669,7 +1680,7 @@ def contract_command( @app.command("reference") @cli_contract.record(cli_contract.CommandRecord( path="docs reference", - summary="Regenerate the human command reference under `docs/reference/`.", + summary="Regenerate the human reference under `docs/reference/`.", synopsis=(cli_contract.Variant(usage="docs reference [--apply]"),), properties=cli_contract.Properties( effect=cli_contract.Effect.WRITE, @@ -1685,6 +1696,14 @@ def contract_command( "`docs/reference/commands/README.md` (``), and " "the gates and exit-42 causes in `docs/reference/exit-codes.md` " "(``).", + "Also, in the origin repo only, the skill catalogue `docs/reference/skills.md` " + "(``: one row per shipped `instructions//SKILL.md`, " + "its description split at the sentence starting \"Use when\") and the page types " + "`docs/reference/page-types.md` (``: one section per " + "`root: kb` type-spec, from what `types describe` reads). Both describe the stack's " + "shipped defaults. In an instance - a tree with `.wikitool-release.json` - they are left " + "alone and the output says so: an instance's adopted type-specs and own instructions may " + "differ, and rewriting a shipped page there would block the next `dist upgrade`.", "Renders each command's summary, synopsis, options, examples, notes, properties in words, " "exit codes with their causes and see-also - but not the ON FAILURE reactions or NEVER, " "which address an agent; `tools/CONTRACT.md` keeps those.", @@ -1716,10 +1735,18 @@ def contract_command( def reference_command( apply: bool = typer.Option(False, "--apply", help="Write changes; default is dry-run (preview only)"), ): - """Regenerate the human command reference under `docs/reference/`.""" + """Regenerate the human reference under `docs/reference/`.""" missing = [] changed = [] - for region in reference_doc.regions(): + generated, left_alone = reference_doc.applicable(reference_doc.regions()) + if left_alone: + typer.echo( + "Left alone in this instance: " + + ", ".join(region.path for region in left_alone) + + " - they describe the stack's shipped defaults; `wikitool types describe ` " + "and `wikitool instructions list` show this instance's own." + ) + for region in generated: path = config.ROOT / region.path if not path.is_file(): missing.append(f"{region.path} (create it with `{blocks.open_marker(region.name)}`)") diff --git a/tools/chemenu/reference_doc.py b/tools/chemenu/reference_doc.py index db33ef4..a0d2244 100644 --- a/tools/chemenu/reference_doc.py +++ b/tools/chemenu/reference_doc.py @@ -17,6 +17,16 @@ person. `regions()` is the one list both `docs reference` and `docs verify` walk; a further generated reference page is one more entry in it. + +Two of its pages - the skill catalogue and the page types - render material an +instance may change: it adopts the page type-specs from their `.template` and +edits them, and it may add instructions of its own. Shown as the instance's +state, `docs reference --apply` would have to rewrite a shipped file there, and +the next `dist upgrade` would then block on it as locally modified. So both +pages describe the stack's shipped defaults and are `origin_only`: generated +and checked in the origin repo, where `types/*.md` is exactly what ships as +`.template`, and left alone in an instance - one carrying a release stamp, +the same line the telemetry default draws. """ from __future__ import annotations @@ -25,25 +35,46 @@ from pathlib import Path from typing import Callable, NamedTuple, Optional from chemenu import blocks, cli_contract, config, toc +from chemenu import version as version_mod REFERENCE_DIR = "docs/reference" COMMANDS_DIR = f"{REFERENCE_DIR}/commands" COMMANDS_INDEX = f"{COMMANDS_DIR}/README.md" EXIT_CODES = f"{REFERENCE_DIR}/exit-codes.md" +SKILLS = f"{REFERENCE_DIR}/skills.md" +PAGE_TYPES = f"{REFERENCE_DIR}/page-types.md" COMMANDS_REGION = "reference-commands" INDEX_REGION = "reference-commands-index" EXIT_CODES_REGION = "reference-exit-codes" +SKILLS_REGION = "reference-skills" +PAGE_TYPES_REGION = "reference-page-types" _SLUG_RE = re.compile(r"[^a-z0-9]+") class Region(NamedTuple): """One generated region: the file it lives in (root-relative), the region's - name, and what renders its content.""" + name, what renders its content, and whether it is generated and checked + only in the origin repo (see the module docstring).""" path: str name: str render: Callable[[], str] + origin_only: bool = False + + +def in_instance() -> bool: + """Whether this tree is an installed instance rather than the origin repo: + `dist export` writes the release stamp, the origin never carries one.""" + return version_mod.stamp_file().is_file() + + +def applicable(found: list[Region]) -> tuple[list[Region], list[Region]]: + """`found` split into the regions this tree generates and checks, and the + `origin_only` ones it leaves alone because it is an instance.""" + if not in_instance(): + return list(found), [] + return [r for r in found if not r.origin_only], [r for r in found if r.origin_only] def group_slug(title: str) -> str: @@ -197,6 +228,154 @@ def render_exit_codes( return "\n".join(lines).rstrip() + "\n" +# --------------------------------------------------------------------------- +# The skill catalogue and the page types - the stack's shipped defaults +# --------------------------------------------------------------------------- + + +class Skill(NamedTuple): + name: str + description: str + + +def shipped_skills() -> list[Skill]: + """Every skill `dist export` ships: `instructions//SKILL.md`, not the + dev-only ones under `instructions/dev/`. Name and description are the + frontmatter the harness itself reads.""" + from chemenu.commands import instructions_cmd # deferred: it imports docs_verify, which imports this + from chemenu.frontmatter_io import read_page + + skills = [] + for directory in instructions_cmd.skill_dirs(): + if instructions_cmd.is_dev_only(directory): + continue + frontmatter, _ = read_page(directory / instructions_cmd.SKILL_FILE) + skills.append(Skill( + str(frontmatter.get("name") or directory.name), + " ".join(str(frontmatter.get("description") or "").split()), + )) + return sorted(skills) + + +# A skill description states what the skill does, then when to use it, in one +# sentence starting "Use when" - the part a harness matches a request against. +_TRIGGER_RE = re.compile(r"\s*\bUse when\b\s*") + + +def split_description(description: str) -> tuple[str, str]: + """A skill description as (what it does, when to use it). A description + with no "Use when" sentence is a skill nobody's request triggers: it runs + only when asked for by name.""" + parts = _TRIGGER_RE.split(description, maxsplit=1) + if len(parts) == 1: + return description, "only when asked for by name" + return parts[0], parts[1] + + +def _cell(text: str) -> str: + """`text` safe inside a table cell: a pipe would end the cell, and outside + inline code an `` placeholder would be read as an HTML tag and + vanish from the rendered page.""" + parts = text.replace("|", "\\|").split("`") + for i in range(0, len(parts), 2): + parts[i] = parts[i].replace("<", "<").replace(">", ">") + return "`".join(parts) + + +def render_skills(skills: list[Skill]) -> str: + """The region body of `skills.md`: one table row per shipped skill.""" + lines = ["| Skill | What it does | Use it when |", "|---|---|---|"] + for skill in skills: + does, when = split_description(skill.description) + lines.append(f"| `{skill.name}` | {_cell(does)} | {_cell(when)} |") + return "\n".join(lines) + "\n" + + +class Subtype(NamedTuple): + value: str + directory: str + index_title: Optional[str] + + +class PageType(NamedTuple): + """What `page-types.md` says about one type: `described` is exactly what + `types describe --json` prints for it, plus where each subtype goes.""" + described: dict + subtypes: tuple[Subtype, ...] + + +def page_types() -> list[PageType]: + """Every page type-spec - the `root: kb` ones `dist export` ships as + `.template` - as `types describe` describes it.""" + from chemenu.commands.dist_cmd import instance_owned_type_stems # deferred: a command module + from chemenu.type_resolver import resolver + from chemenu.types_core import describe_type + + stems = instance_owned_type_stems() + found = [] + for type_path, frontmatter in resolver.list_type_specs(): + if Path(type_path).stem not in stems: + continue + described = describe_type(frontmatter["name"]) + subtypes: tuple[Subtype, ...] = () + field = described["subtype_field"] + if field: + values = next((f["enum"] for f in described["fields"] if f["field"] == field), None) + layout = resolver.get_layout(type_path) or {} + subtypes = tuple( + Subtype( + value, + resolver.subtype_dir(type_path, value) or "", + layout.get(value, {}).get("title"), + ) + for value in (values or list(layout)) + ) + found.append(PageType(described, subtypes)) + return found + + +def _type_dir(described: dict) -> str: + return f"{described['root']}/{described['base_dir']}" + + +def render_page_type(page_type: PageType) -> str: + """One type's section on `page-types.md`.""" + described = page_type.described + base = _type_dir(described) + lines = [f"## `{described['name']}`", "", described["description"] or "", ""] + if page_type.subtypes: + lines.append(f"- Written to: `{base}//`") + else: + lines.append(f"- Written to: `{base}/`") + if described["title_prefix"]: + lines.append(f"- Title prefix: `{described['title_prefix']}`") + lines.append("") + + if page_type.subtypes: + lines += [f"**Subtypes** (field `{described['subtype_field']}`)", ""] + lines += ["| Value | Written to | Index section |", "|---|---|---|"] + for subtype in page_type.subtypes: + lines.append( + f"| `{subtype.value}` | `{base}/{subtype.directory}/` | {subtype.index_title or ''} |" + ) + lines.append("") + + lines += ["**Required fields**", ""] + for field in described["fields"]: + if not field["required"]: + continue + kind = field["type"] or "any" + if field["enum"]: + kind += ", one of " + ", ".join(f"`{value}`" for value in field["enum"]) + lines.append(f"- `{field['field']}` - {kind}") + return "\n".join(lines).rstrip() + "\n" + + +def render_page_types(types: list[PageType]) -> str: + """The region body of `page-types.md`: one section per page type.""" + return "\n".join(render_page_type(t) for t in types).rstrip() + "\n" + + # --------------------------------------------------------------------------- # The generated regions, and writing one # --------------------------------------------------------------------------- @@ -225,6 +404,10 @@ def regions( ] found.append(Region(COMMANDS_INDEX, INDEX_REGION, lambda: render_index(records, groups))) found.append(Region(EXIT_CODES, EXIT_CODES_REGION, lambda: render_exit_codes(records, groups))) + # Looked up at render time, not bound here, so a fixture tree can stand in + # its own skills and types for the real ones. + found.append(Region(SKILLS, SKILLS_REGION, lambda: render_skills(shipped_skills()), origin_only=True)) + found.append(Region(PAGE_TYPES, PAGE_TYPES_REGION, lambda: render_page_types(page_types()), origin_only=True)) return found diff --git a/tools/chemenu/tests/test_reference_doc.py b/tools/chemenu/tests/test_reference_doc.py index 6c5ea87..9eed31b 100644 --- a/tools/chemenu/tests/test_reference_doc.py +++ b/tools/chemenu/tests/test_reference_doc.py @@ -1,11 +1,14 @@ +import json import re import pytest from typer.testing import CliRunner from chemenu import blocks, cli, cli_contract, config, reference_doc, toc +from chemenu import version as version_mod from chemenu.cli import app -from chemenu.commands import dist_cmd, docs_verify +from chemenu.commands import dist_cmd, docs_verify, instructions_cmd +from chemenu.type_resolver import resolver runner = CliRunner() @@ -94,6 +97,112 @@ def test_the_reference_pages_survive_dist_export_unchanged(): assert plan[region.path].content == (config.ROOT / region.path).read_text(encoding="utf-8") +def _type_sections(page: str) -> dict[str, str]: + region = blocks.find(page, reference_doc.PAGE_TYPES_REGION) + return { + match.group(1): match.group(2) + for match in re.finditer(r"^## `([^`]+)`\n(.*?)(?=^## |\Z)", region, re.M | re.S) + } + + +def test_the_skill_catalogue_has_one_row_per_shipped_skill(): + shipped = { + d.name for d in instructions_cmd.skill_dirs() if not instructions_cmd.is_dev_only(d) + } + assert shipped + region = blocks.find( + (config.ROOT / reference_doc.SKILLS).read_text(encoding="utf-8"), reference_doc.SKILLS_REGION + ) + rows = re.findall(r"^\| `([^`]+)` \|", region, re.M) + assert sorted(rows) == sorted(shipped) + for directory in instructions_cmd.skill_dirs(): + if instructions_cmd.is_dev_only(directory): + assert f"`{directory.name}`" not in region + + +def test_page_types_has_one_section_per_page_type_matching_types_describe(): + """Required fields and subtypes are what `types describe --json` says, and + each subtype's directory is where `new` would write it.""" + sections = _type_sections((config.ROOT / reference_doc.PAGE_TYPES).read_text(encoding="utf-8")) + names = { + fm["name"] for path, fm in resolver.list_type_specs() + if path.rsplit("/", 1)[-1][:-3] in dist_cmd.instance_owned_type_stems() + } + assert names and set(sections) == names + + for name, section in sections.items(): + result = runner.invoke(app, ["types", "describe", name, "--json"]) + assert result.exit_code == 0, result.output + described = json.loads(result.output) + required = [f["field"] for f in described["fields"] if f["required"]] + listed = re.findall(r"^- `([^`]+)` - ", section.split("**Required fields**")[1], re.M) + assert listed == required, name + + field = described["subtype_field"] + rows = re.findall(r"^\| `([^`]+)` \| `([^`]+)` \|", section, re.M) + if not field: + assert rows == [], name + continue + enum = next(f["enum"] for f in described["fields"] if f["field"] == field) + assert [value for value, _ in rows] == enum, name + for value, written_to in rows: + target = resolver.compute_target_dir(described["type_path"], {field: value}) + assert written_to == target.relative_to(config.ROOT).as_posix() + "/", (name, value) + + +def test_a_description_is_split_at_its_use_when_sentence(): + assert reference_doc.split_description("Does X. Use when Y happens.") == ("Does X.", "Y happens.") + assert reference_doc.split_description("Does X.") == ("Does X.", "only when asked for by name") + + +def test_a_table_cell_escapes_pipes_and_tags_but_not_inside_code(): + cell = reference_doc._cell('says "ingest " or `a | c` | d') + assert cell == 'says "ingest <file>" or `a \\| c` \\| d' + + +@pytest.fixture +def exported_instance(tmp_path, monkeypatch): + """A `dist export` of this tree, adopted and then changed the way an + instance may: a type-spec of its own making, and a skill of its own.""" + target = tmp_path / "instance" + dist_cmd.run_export(target) + monkeypatch.setattr(config, "ROOT", target) + monkeypatch.setattr(resolver, "_repo_root", target) + for template in sorted((target / "types").glob("*.template")): + template.rename(template.with_suffix("")) + entity = target / "types" / "entity.md" + entity.write_text( + entity.read_text(encoding="utf-8").replace( + "description: Base type for entity pages", "description: This instance's entities" + ), + encoding="utf-8", + ) + own = target / "instructions" / "own-skill" + own.mkdir() + (own / "SKILL.md").write_text( + "---\nname: own-skill\ndescription: Does this instance's thing. Use when asked.\n---\n# Own\n", + encoding="utf-8", + ) + return target + + +def test_an_adapted_instance_leaves_both_pages_alone(exported_instance): + root = exported_instance + assert version_mod.stamp_file().is_file() + pages = { + relative: (root / relative).read_text(encoding="utf-8") + for relative in (reference_doc.SKILLS, reference_doc.PAGE_TYPES) + } + assert docs_verify.check_reference_regions() == [] + + result = runner.invoke(app, ["docs", "reference", "--apply"]) + assert result.exit_code == 0, result.output + assert "Left alone in this instance" in result.output + for relative, before in pages.items(): + assert (root / relative).read_text(encoding="utf-8") == before, relative + assert docs_verify.check_reference_regions() == [] + + # --- a fixture tree ------------------------------------------------------- GROUPS = (("Fixture things", ("frobnicate",)), ("Other", ("gate keep",))) @@ -127,15 +236,34 @@ HEADS = { "docs/reference/commands/other.md": "# Other\n\nThe other one.\n", "docs/reference/commands/README.md": "# Command reference\n\nAll of it.\n", "docs/reference/exit-codes.md": "# Exit codes and gates\n\nWhere it stops.\n", + "docs/reference/skills.md": "# Skill catalogue\n\nThe skills.\n", + "docs/reference/page-types.md": "# Page types\n\nThe types.\n", } REGION_OF = { "docs/reference/commands/fixture-things.md": reference_doc.COMMANDS_REGION, "docs/reference/commands/other.md": reference_doc.COMMANDS_REGION, "docs/reference/commands/README.md": reference_doc.INDEX_REGION, "docs/reference/exit-codes.md": reference_doc.EXIT_CODES_REGION, + "docs/reference/skills.md": reference_doc.SKILLS_REGION, + "docs/reference/page-types.md": reference_doc.PAGE_TYPES_REGION, } +def _page_type(description: str = "A fixture type.") -> reference_doc.PageType: + return reference_doc.PageType( + described={ + "name": "gadget", "description": description, "subtype_field": "gadget_type", + "root": "kb", "base_dir": "gadgets", "title_prefix": None, + "fields": [ + {"field": "type", "required": True, "type": "string", "enum": None}, + {"field": "gadget_type", "required": True, "type": "string", "enum": ["widget"]}, + {"field": "tags", "required": False, "type": "array", "enum": None}, + ], + }, + subtypes=(reference_doc.Subtype("widget", "widgets", "Widgets"),), + ) + + @pytest.fixture def tree(tmp_path, monkeypatch): monkeypatch.setattr(config, "ROOT", tmp_path) @@ -143,6 +271,11 @@ def tree(tmp_path, monkeypatch): monkeypatch.setattr(cli_contract, "all_records", lambda: dict(records)) monkeypatch.setattr(cli_contract, "GROUPS", GROUPS) monkeypatch.setattr(cli, "options_texts", lambda paths: {p: " --flag A flag." for p in paths}) + monkeypatch.setattr( + reference_doc, "shipped_skills", + lambda: [reference_doc.Skill("fixture-skill", "Does a thing. Use when asked.")], + ) + monkeypatch.setattr(reference_doc, "page_types", lambda: [_page_type()]) for relative, head in HEADS.items(): path = tmp_path / relative path.parent.mkdir(parents=True, exist_ok=True) @@ -286,3 +419,44 @@ def test_the_index_links_every_command_to_its_section(tree): index = (root / "docs/reference/commands/README.md").read_text(encoding="utf-8") assert "- [`frobnicate`](fixture-things.md#frobnicate) - Does a thing." in index assert "- [`gate keep`](other.md#gate-keep) - Does a thing." in index + + +def test_a_changed_skill_or_type_makes_verify_name_its_page(tree, monkeypatch): + runner.invoke(app, ["docs", "reference", "--apply"]) + assert docs_verify.check_reference_regions() == [] + + monkeypatch.setattr( + reference_doc, "shipped_skills", + lambda: [reference_doc.Skill("fixture-skill", "Does another thing. Use when asked.")], + ) + monkeypatch.setattr(reference_doc, "page_types", lambda: [_page_type("A changed type.")]) + issues = docs_verify.check_reference_regions() + for page in ("docs/reference/skills.md", "docs/reference/page-types.md"): + assert any(page in issue and "docs reference --apply" in issue for issue in issues), page + + +def test_the_fixture_skill_and_type_render(tree): + root, _ = tree + runner.invoke(app, ["docs", "reference", "--apply"]) + skills = (root / "docs/reference/skills.md").read_text(encoding="utf-8") + assert "| `fixture-skill` | Does a thing. | asked. |" in skills + types = (root / "docs/reference/page-types.md").read_text(encoding="utf-8") + assert "- Written to: `kb/gadgets//`" in types + assert "| `widget` | `kb/gadgets/widgets/` | Widgets |" in types + assert "- `gadget_type` - string, one of `widget`" in types + assert "`tags`" not in types + + +def test_an_instance_skips_a_missing_skill_or_type_page(tree): + """In an instance the two pages are not the tree's to check - not even + whether they exist.""" + root, _ = tree + runner.invoke(app, ["docs", "reference", "--apply"]) + (root / version_mod.RELEASE_STAMP_FILENAME).write_text("{}", encoding="utf-8") + (root / "docs/reference/skills.md").unlink() + (root / "docs/reference/page-types.md").write_text("# Page types\n", encoding="utf-8") + assert docs_verify.check_reference_regions() == [] + result = runner.invoke(app, ["docs", "reference", "--apply"]) + assert result.exit_code == 0, result.output + assert "Left alone in this instance: docs/reference/skills.md, docs/reference/page-types.md" in result.output + assert not (root / "docs/reference/skills.md").exists()