chemenu/tools/chemenu/reference_doc.py

"""The human command reference under `docs/reference/`, generated from the records.

`cli_contract` is the one source of what a command does; `tools/CONTRACT.md`
renders it for an agent, and this module renders it for a person. The two
differ on purpose: a record's ON FAILURE reactions and its NEVER lines tell an
agent what to do ("show the output verbatim and stop"), not a person what
happens, so they stay out here - only each failure's cause is kept, under the
exit code it produces. Properties are spelled out in words rather than as the
index's `budget:counted` tokens.

Every file here is half human, half generated, like `INSTALL.md`'s
prerequisites lists: a title and a one-sentence head someone wrote, then one
`<!-- wikitool:reference-... -->` region `docs reference --apply` rewrites. The
tool never creates a page or places a region - a missing one is reported - so
a group added to `cli_contract.GROUPS` gets its page, and its head, from a
person.

`regions()` is the one list both `docs reference` and `docs verify` walk; a
further generated reference page is one more entry in it.

Two of its pages - the skill catalogue and the page types - render material an
instance may change: it adopts the page type-specs from their `.template` and
edits them, and it may add instructions of its own. Shown as the instance's
state, `docs reference --apply` would have to rewrite a shipped file there, and
the next `dist upgrade` would then block on it as locally modified. So both
pages describe the stack's shipped defaults and are `origin_only`: generated
and checked in the origin repo, where `types/*.md` is exactly what ships as
`.template`, and left alone in an instance - one carrying a release stamp,
the same line the telemetry default draws.

`docs/reference/configuration.md` carries four more regions, one per
hand-written configuration file, rendered from the records in `config_shapes`.
Those records are the tool's own, read by the loaders and shipped with
`tools/`, so their regions are generated and checked in an instance too.
"""
from __future__ import annotations

import re
from pathlib import Path
from typing import Callable, NamedTuple, Optional

from chemenu import blocks, cli_contract, config, config_shapes, toc
from chemenu import version as version_mod

REFERENCE_DIR = "docs/reference"
COMMANDS_DIR = f"{REFERENCE_DIR}/commands"
COMMANDS_INDEX = f"{COMMANDS_DIR}/README.md"
EXIT_CODES = f"{REFERENCE_DIR}/exit-codes.md"
SKILLS = f"{REFERENCE_DIR}/skills.md"
PAGE_TYPES = f"{REFERENCE_DIR}/page-types.md"
CONFIGURATION = f"{REFERENCE_DIR}/configuration.md"

COMMANDS_REGION = "reference-commands"
INDEX_REGION = "reference-commands-index"
EXIT_CODES_REGION = "reference-exit-codes"
SKILLS_REGION = "reference-skills"
PAGE_TYPES_REGION = "reference-page-types"

# One region per hand-written configuration file, each placed inside the
# section of `configuration.md` that carries the file's behaviour in prose.
# Rendered from the `config_shapes` record named beside it, which ships with
# `tools/`, so unlike the two pages above they are generated and checked in an
# instance as well.
CONFIG_REGIONS = (
    ("reference-config-telemetry", "TELEMETRY"),
    ("reference-config-remotes", "REMOTES"),
    ("reference-config-tasks", "TASKS"),
    ("reference-config-superproductivity", "SUPERPRODUCTIVITY"),
    ("reference-config-caldav", "CALDAV"),
    ("reference-config-upload", "UPLOAD"),
)

_SLUG_RE = re.compile(r"[^a-z0-9]+")


class Region(NamedTuple):
    """One generated region: the file it lives in (root-relative), the region's
    name, what renders its content, and whether it is generated and checked
    only in the origin repo (see the module docstring)."""
    path: str
    name: str
    render: Callable[[], str]
    origin_only: bool = False


def in_instance() -> bool:
    """Whether this tree is an installed instance rather than the origin repo:
    `dist export` writes the release stamp, the origin never carries one."""
    return version_mod.stamp_file().is_file()


def applicable(found: list[Region]) -> tuple[list[Region], list[Region]]:
    """`found` split into the regions this tree generates and checks, and the
    `origin_only` ones it leaves alone because it is an instance."""
    if not in_instance():
        return list(found), []
    return [r for r in found if not r.origin_only], [r for r in found if r.origin_only]


def group_slug(title: str) -> str:
    """The page name of a `GROUPS` entry: lower case, every run of anything
    else than a letter or digit one hyphen (`Types, instructions and docs` ->
    `types-instructions-and-docs`)."""
    return _SLUG_RE.sub("-", title.lower()).strip("-")


def group_page(title: str) -> str:
    return f"{COMMANDS_DIR}/{group_slug(title)}.md"


def command_anchor(path: str) -> str:
    """The anchor of a command's `## `<path>`` heading on its group page."""
    return toc.anchor(f"`{path}`")


# ---------------------------------------------------------------------------
# Rendering
# ---------------------------------------------------------------------------

_BUDGET_WORDS = {
    cli_contract.Budget.COUNTED: "yes",
    cli_contract.Budget.EXEMPT: "no",
    cli_contract.Budget.EXEMPT_WITHOUT_ARGS: "only when called with arguments",
}


def _yes_no(value: bool) -> str:
    return "yes" if value else "no"


def properties_lines(props: cli_contract.Properties) -> list[str]:
    """A record's properties in words, one list item each."""
    lines = [
        f"- Changes files: {_yes_no(props.effect == cli_contract.Effect.WRITE)}",
        f"- Idempotent: {_yes_no(props.idempotent == cli_contract.Idempotent.YES)}",
        f"- Atomic: {props.atomic}",
        f"- Counts toward the session call budget: {_BUDGET_WORDS[props.budget]}",
        f"- Can reach the network: {_yes_no(props.network == cli_contract.Network.YES)}",
    ]
    if props.gates:
        lines.append(
            f"- Gates: {', '.join(props.gates)} - see [exit codes and gates](../exit-codes.md)"
        )
    return lines


def render_command(rec: cli_contract.CommandRecord, options_text: str = "") -> str:
    """One command's section on its group page."""
    lines = [f"## `{rec.path}`", "", rec.summary, ""]

    lines += ["**Synopsis**", ""]
    for variant in rec.synopsis:
        note = f" - {variant.notes}" if variant.notes else ""
        lines.append(f"- `wikitool {variant.usage}`{note}")
    lines.append("")

    if options_text.strip():
        lines += ["**Options**", "", "```text", options_text.rstrip(), "```", ""]

    if rec.examples:
        lines += ["**Examples**", "", "```bash", *rec.examples, "```", ""]

    lines += ["**Behaviour**", ""]
    lines += [f"- {note}" for note in rec.notes]
    lines.append("")

    lines += ["**Properties**", "", *properties_lines(rec.properties), ""]

    lines += ["**Exit codes**", ""]
    lines += [f"- `{code}` - {cause}" for code, cause in cli_contract.exit_status_entries(rec)]
    lines.append("")

    if rec.see_also:
        lines += ["**See also**", ""]
        lines += [f"- {item}" for item in rec.see_also]
        lines.append("")

    return "\n".join(lines).rstrip() + "\n"


def render_group(
    paths: tuple[str, ...],
    records: dict[str, cli_contract.CommandRecord],
    options: dict[str, str],
) -> str:
    """The region body of one group page: its commands, in `GROUPS` order."""
    return "\n".join(
        render_command(records[path], options.get(path, "")) for path in paths if path in records
    ).rstrip() + "\n"


def render_index(
    records: dict[str, cli_contract.CommandRecord],
    groups: cli_contract.GroupsType,
) -> str:
    """The region body of the commands README: every group, every command,
    its one-sentence summary, linked to its section."""
    lines: list[str] = []
    for title, paths in groups:
        page = f"{group_slug(title)}.md"
        lines += [f"## {title}", "", f"Full records: [{page}]({page})", ""]
        for path in paths:
            if path in records:
                lines.append(
                    f"- [`{path}`]({page}#{command_anchor(path)}) - {records[path].summary}"
                )
        lines.append("")
    return "\n".join(lines).rstrip() + "\n"


def _command_link(path: str, groups: cli_contract.GroupsType) -> str:
    title = cli_contract.group_of(path, groups)
    if title is None:
        return f"`{path}`"
    return f"[`{path}`](commands/{group_slug(title)}.md#{command_anchor(path)})"


def render_exit_codes(
    records: dict[str, cli_contract.CommandRecord],
    groups: cli_contract.GroupsType,
) -> str:
    """The region body of `exit-codes.md`: which command declares which gate,
    and every cause of an exit 42. What an exit code means is linked, not
    repeated - `AGENTS.md` is where that rule lives."""
    ordered = [path for path in cli_contract.grouped_paths(groups) if path in records]
    lines = [
        "What each exit code means, and how a caller reacts to it, is "
        "[AGENTS.md § Tool error contract](../../AGENTS.md#tool-error-contract). What a gate is "
        "and why it is enforced in code is [AGENTS.md § Gates](../../AGENTS.md#gates). This page "
        "lists which commands declare a gate, and every cause for which a command exits 42.",
        "",
        "## Gates by command",
        "",
    ]
    gates: dict[str, list[str]] = {}
    for path in ordered:
        for gate in records[path].properties.gates:
            gates.setdefault(gate, []).append(path)
    for gate in sorted(gates):
        lines += [f"### {gate}", ""]
        lines += [f"- {_command_link(path, groups)}" for path in gates[gate]]
        lines.append("")

    lines += ["## Exit 42 causes", ""]
    for path in ordered:
        causes = [cause for code, cause in cli_contract.exit_status_entries(records[path]) if code == 42]
        lines += [f"- {_command_link(path, groups)}: {cause}" for cause in causes]
    return "\n".join(lines).rstrip() + "\n"


# ---------------------------------------------------------------------------
# The skill catalogue and the page types - the stack's shipped defaults
# ---------------------------------------------------------------------------


class Skill(NamedTuple):
    name: str
    description: str


def shipped_skills() -> list[Skill]:
    """Every skill `dist export` ships: `instructions/<name>/SKILL.md`, not the
    dev-only ones under `instructions/dev/`. Name and description are the
    frontmatter the harness itself reads."""
    from chemenu.commands import instructions_cmd  # deferred: it imports docs_verify, which imports this
    from chemenu.frontmatter_io import read_page

    skills = []
    for directory in instructions_cmd.skill_dirs():
        if instructions_cmd.is_dev_only(directory):
            continue
        frontmatter, _ = read_page(directory / instructions_cmd.SKILL_FILE)
        skills.append(Skill(
            str(frontmatter.get("name") or directory.name),
            " ".join(str(frontmatter.get("description") or "").split()),
        ))
    return sorted(skills)


# A skill description states what the skill does, then when to use it, in one
# sentence starting "Use when" - the part a harness matches a request against.
_TRIGGER_RE = re.compile(r"\s*\bUse when\b\s*")


def split_description(description: str) -> tuple[str, str]:
    """A skill description as (what it does, when to use it). A description
    with no "Use when" sentence is a skill nobody's request triggers: it runs
    only when asked for by name."""
    parts = _TRIGGER_RE.split(description, maxsplit=1)
    if len(parts) == 1:
        return description, "only when asked for by name"
    return parts[0], parts[1]


def _cell(text: str) -> str:
    """`text` safe inside a table cell: a pipe would end the cell, and outside
    inline code an `<file>` placeholder would be read as an HTML tag and
    vanish from the rendered page."""
    parts = text.replace("|", "\\|").split("`")
    for i in range(0, len(parts), 2):
        parts[i] = parts[i].replace("<", "&lt;").replace(">", "&gt;")
    return "`".join(parts)


def render_skills(skills: list[Skill]) -> str:
    """The region body of `skills.md`: one table row per shipped skill."""
    lines = ["| Skill | What it does | Use it when |", "|---|---|---|"]
    for skill in skills:
        does, when = split_description(skill.description)
        lines.append(f"| `{skill.name}` | {_cell(does)} | {_cell(when)} |")
    return "\n".join(lines) + "\n"


class Subtype(NamedTuple):
    value: str
    directory: str
    index_title: Optional[str]


class PageType(NamedTuple):
    """What `page-types.md` says about one type: `described` is exactly what
    `types describe --json` prints for it, plus where each subtype goes."""
    described: dict
    subtypes: tuple[Subtype, ...]


def page_types() -> list[PageType]:
    """Every page type-spec - the `root: kb` ones `dist export` ships as
    `.template` - as `types describe` describes it."""
    from chemenu.commands.dist_cmd import instance_owned_type_stems  # deferred: a command module
    from chemenu.type_resolver import resolver
    from chemenu.types_core import describe_type

    stems = instance_owned_type_stems()
    found = []
    for type_path, frontmatter in resolver.list_type_specs():
        if Path(type_path).stem not in stems:
            continue
        described = describe_type(frontmatter["name"])
        subtypes: tuple[Subtype, ...] = ()
        field = described["subtype_field"]
        if field:
            values = next((f["enum"] for f in described["fields"] if f["field"] == field), None)
            layout = resolver.get_layout(type_path) or {}
            subtypes = tuple(
                Subtype(
                    value,
                    resolver.subtype_dir(type_path, value) or "",
                    layout.get(value, {}).get("title"),
                )
                for value in (values or list(layout))
            )
        found.append(PageType(described, subtypes))
    return found


def _type_dir(described: dict) -> str:
    return f"{described['root']}/{described['base_dir']}"


def render_page_type(page_type: PageType) -> str:
    """One type's section on `page-types.md`."""
    described = page_type.described
    base = _type_dir(described)
    lines = [f"## `{described['name']}`", "", described["description"] or "", ""]
    if page_type.subtypes:
        lines.append(f"- Written to: `{base}/<subtype directory>/`")
    else:
        lines.append(f"- Written to: `{base}/`")
    if described["title_prefix"]:
        lines.append(f"- Title prefix: `{described['title_prefix']}`")
    lines.append("")

    if page_type.subtypes:
        lines += [f"**Subtypes** (field `{described['subtype_field']}`)", ""]
        lines += ["| Value | Written to | Index section |", "|---|---|---|"]
        for subtype in page_type.subtypes:
            lines.append(
                f"| `{subtype.value}` | `{base}/{subtype.directory}/` | {subtype.index_title or ''} |"
            )
        lines.append("")

    lines += ["**Required fields**", ""]
    for field in described["fields"]:
        if not field["required"]:
            continue
        kind = field["type"] or "any"
        if field["enum"]:
            kind += ", one of " + ", ".join(f"`{value}`" for value in field["enum"])
        lines.append(f"- `{field['field']}` - {kind}")
    return "\n".join(lines).rstrip() + "\n"


def render_page_types(types: list[PageType]) -> str:
    """The region body of `page-types.md`: one section per page type."""
    return "\n".join(render_page_type(t) for t in types).rstrip() + "\n"


# ---------------------------------------------------------------------------
# The generated regions, and writing one
# ---------------------------------------------------------------------------


def regions(
    records: Optional[dict[str, cli_contract.CommandRecord]] = None,
    groups: Optional[cli_contract.GroupsType] = None,
    options: Optional[dict[str, str]] = None,
) -> list[Region]:
    """Every generated region under `docs/reference/`, in a stable order.

    `options` defaults to the real CLI's rendered OPTIONS blocks; a test
    passing its own records passes its own options too (or `{}`)."""
    records = records if records is not None else cli_contract.all_records()
    groups = groups if groups is not None else cli_contract.GROUPS
    if options is None:
        from chemenu import cli  # deferred: `cli` imports every command module, this one's caller included

        options = cli.options_texts([p for p in cli_contract.grouped_paths(groups) if p in records])

    found = [
        Region(group_page(title), COMMANDS_REGION,
               lambda paths=paths: render_group(paths, records, options))
        for title, paths in groups
    ]
    found.append(Region(COMMANDS_INDEX, INDEX_REGION, lambda: render_index(records, groups)))
    found.append(Region(EXIT_CODES, EXIT_CODES_REGION, lambda: render_exit_codes(records, groups)))
    # Looked up at render time, not bound here, so a fixture tree can stand in
    # its own skills and types for the real ones.
    found.append(Region(SKILLS, SKILLS_REGION, lambda: render_skills(shipped_skills()), origin_only=True))
    found.append(Region(PAGE_TYPES, PAGE_TYPES_REGION, lambda: render_page_types(page_types()), origin_only=True))
    # The record is looked up at render time, so a test replacing one sees it.
    found += [
        Region(CONFIGURATION, name,
               lambda attr=shape_name: config_shapes.render_markdown(getattr(config_shapes, attr)))
        for name, shape_name in CONFIG_REGIONS
    ]
    return found


def refresh(text: str, region: Region) -> Optional[str]:
    """`text` with `region` regenerated and its table of contents brought in
    line, or None when the file carries no such region to fill.

    The TOC is refreshed in the same step because the generated `##` headings
    are what it lists - without it, every regeneration that added a command
    would leave `docs verify` reporting a stale TOC until `docs toc --apply`
    ran as well."""
    if blocks.find(text, region.name) is None:
        return None
    content = region.render().strip("\n")
    # The trailing newline keeps a blank line between the region and prose
    # that follows it; `blocks.replace` was written for regions that end a
    # page, and the configuration page is the first to carry them mid-page.
    wrapped = blocks.open_marker(region.name) + "\n" + content + "\n" + blocks.close_marker(region.name) + "\n"
    return toc.upsert(blocks.replace(text, region.name, wrapped))


def orphan_pages(groups: Optional[cli_contract.GroupsType] = None) -> list[Path]:
    """Pages under `docs/reference/commands/` no group in `groups` names."""
    groups = groups if groups is not None else cli_contract.GROUPS
    directory = config.ROOT / COMMANDS_DIR
    if not directory.is_dir():
        return []
    expected = {Path(group_page(title)).name for title, _ in groups} | {Path(COMMANDS_INDEX).name}
    return sorted(p for p in directory.glob("*.md") if p.name not in expected)