torbenandClaude Opus 5.5
070cae6aad
Files changed: - CHANGES.md - README.md - VERSION - docs/reference/README.md - docs/reference/commands/README.md - docs/reference/commands/types-instructions-and-docs.md - docs/reference/page-types.md - docs/reference/skills.md - instructions/dev/doc-pull-through.md - tools/CONTRACT.md - tools/README.md - tools/chemenu/commands/docs_verify.py - tools/chemenu/reference_doc.py - tools/chemenu/tests/test_reference_doc.py Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
463 lines
20 KiB
Python
463 lines
20 KiB
Python
import json
|
|
import re
|
|
|
|
import pytest
|
|
from typer.testing import CliRunner
|
|
|
|
from chemenu import blocks, cli, cli_contract, config, reference_doc, toc
|
|
from chemenu import version as version_mod
|
|
from chemenu.cli import app
|
|
from chemenu.commands import dist_cmd, docs_verify, instructions_cmd
|
|
from chemenu.type_resolver import resolver
|
|
|
|
runner = CliRunner()
|
|
|
|
# --- the real tree --------------------------------------------------------
|
|
|
|
|
|
def _group_pages() -> dict[str, str]:
|
|
return {
|
|
title: (config.ROOT / reference_doc.group_page(title)).read_text(encoding="utf-8")
|
|
for title, _ in cli_contract.GROUPS
|
|
}
|
|
|
|
|
|
def test_the_real_reference_is_current():
|
|
"""What a session that changed a record and forgot `docs reference --apply`
|
|
is caught by."""
|
|
assert docs_verify.check_reference_regions() == []
|
|
|
|
|
|
def test_every_group_has_one_page_and_every_command_one_section():
|
|
pages = _group_pages()
|
|
for path in cli_contract.all_records():
|
|
heading = f"## `{path}`\n"
|
|
hits = [title for title, text in pages.items() if text.count(heading) == 1]
|
|
assert len(hits) == 1, path
|
|
assert sum(text.count(heading) for text in pages.values()) == 1, path
|
|
|
|
|
|
def test_group_slugs_are_unique():
|
|
slugs = [reference_doc.group_slug(title) for title, _ in cli_contract.GROUPS]
|
|
assert len(slugs) == len(set(slugs))
|
|
|
|
|
|
def test_no_page_carries_an_agent_addressed_field():
|
|
"""ON FAILURE reactions and NEVER tell an agent what to do; the human
|
|
reference keeps only the cause of each failure."""
|
|
text = "\n".join(_group_pages().values())
|
|
text += (config.ROOT / reference_doc.EXIT_CODES).read_text(encoding="utf-8")
|
|
for rec in cli_contract.all_records().values():
|
|
for failure in rec.failures:
|
|
if failure.reaction:
|
|
assert failure.reaction not in text, (rec.path, failure.reaction)
|
|
for line in rec.never:
|
|
assert line not in text, (rec.path, line)
|
|
|
|
|
|
def test_options_match_the_options_section_of_h():
|
|
texts = cli.options_texts(sorted(cli_contract.all_records()))
|
|
for path, options in texts.items():
|
|
result = runner.invoke(app, [*path.split(" "), "-h"])
|
|
assert result.exit_code == 0, (path, result.output)
|
|
if options.strip():
|
|
assert f"OPTIONS\n{options.rstrip()}\n\n" in result.output, path
|
|
else:
|
|
assert "\nOPTIONS\n" not in result.output, path
|
|
|
|
|
|
def _section(markdown: str, heading: str) -> str:
|
|
start = markdown.index(f"## {heading}\n")
|
|
end = markdown.find("\n## ", start + 1)
|
|
return markdown[start:end if end != -1 else None]
|
|
|
|
|
|
def test_exit_codes_names_every_gate_and_links_rather_than_repeats_agents_md():
|
|
page = (config.ROOT / reference_doc.EXIT_CODES).read_text(encoding="utf-8")
|
|
for rec in cli_contract.all_records().values():
|
|
for gate in rec.properties.gates:
|
|
assert f"### {gate}\n" in page, gate
|
|
assert "(../../AGENTS.md#tool-error-contract)" in page
|
|
assert "(../../AGENTS.md#gates)" in page
|
|
|
|
agents = (config.ROOT / "AGENTS.md").read_text(encoding="utf-8")
|
|
for heading in ("Tool error contract", "Gates"):
|
|
for sentence in re.split(r"(?<=[.:])\s+", _section(agents, heading)):
|
|
sentence = sentence.strip()
|
|
if len(sentence) >= 30:
|
|
assert sentence not in page, sentence
|
|
|
|
|
|
def test_the_reference_pages_survive_dist_export_unchanged():
|
|
"""`dist export` strips `dist:strip` spans from every shipped text file; a
|
|
record quoting the marker pair verbatim would be cut in the exported page
|
|
and the fresh instance's `docs verify` would call it stale."""
|
|
plan = dist_cmd.build_plan()
|
|
for region in reference_doc.regions():
|
|
assert plan[region.path].content == (config.ROOT / region.path).read_text(encoding="utf-8")
|
|
|
|
|
|
def _type_sections(page: str) -> dict[str, str]:
|
|
region = blocks.find(page, reference_doc.PAGE_TYPES_REGION)
|
|
return {
|
|
match.group(1): match.group(2)
|
|
for match in re.finditer(r"^## `([^`]+)`\n(.*?)(?=^## |\Z)", region, re.M | re.S)
|
|
}
|
|
|
|
|
|
def test_the_skill_catalogue_has_one_row_per_shipped_skill():
|
|
shipped = {
|
|
d.name for d in instructions_cmd.skill_dirs() if not instructions_cmd.is_dev_only(d)
|
|
}
|
|
assert shipped
|
|
region = blocks.find(
|
|
(config.ROOT / reference_doc.SKILLS).read_text(encoding="utf-8"), reference_doc.SKILLS_REGION
|
|
)
|
|
rows = re.findall(r"^\| `([^`]+)` \|", region, re.M)
|
|
assert sorted(rows) == sorted(shipped)
|
|
for directory in instructions_cmd.skill_dirs():
|
|
if instructions_cmd.is_dev_only(directory):
|
|
assert f"`{directory.name}`" not in region
|
|
|
|
|
|
def test_page_types_has_one_section_per_page_type_matching_types_describe():
|
|
"""Required fields and subtypes are what `types describe --json` says, and
|
|
each subtype's directory is where `new` would write it."""
|
|
sections = _type_sections((config.ROOT / reference_doc.PAGE_TYPES).read_text(encoding="utf-8"))
|
|
names = {
|
|
fm["name"] for path, fm in resolver.list_type_specs()
|
|
if path.rsplit("/", 1)[-1][:-3] in dist_cmd.instance_owned_type_stems()
|
|
}
|
|
assert names and set(sections) == names
|
|
|
|
for name, section in sections.items():
|
|
result = runner.invoke(app, ["types", "describe", name, "--json"])
|
|
assert result.exit_code == 0, result.output
|
|
described = json.loads(result.output)
|
|
required = [f["field"] for f in described["fields"] if f["required"]]
|
|
listed = re.findall(r"^- `([^`]+)` - ", section.split("**Required fields**")[1], re.M)
|
|
assert listed == required, name
|
|
|
|
field = described["subtype_field"]
|
|
rows = re.findall(r"^\| `([^`]+)` \| `([^`]+)` \|", section, re.M)
|
|
if not field:
|
|
assert rows == [], name
|
|
continue
|
|
enum = next(f["enum"] for f in described["fields"] if f["field"] == field)
|
|
assert [value for value, _ in rows] == enum, name
|
|
for value, written_to in rows:
|
|
target = resolver.compute_target_dir(described["type_path"], {field: value})
|
|
assert written_to == target.relative_to(config.ROOT).as_posix() + "/", (name, value)
|
|
|
|
|
|
def test_a_description_is_split_at_its_use_when_sentence():
|
|
assert reference_doc.split_description("Does X. Use when Y happens.") == ("Does X.", "Y happens.")
|
|
assert reference_doc.split_description("Does X.") == ("Does X.", "only when asked for by name")
|
|
|
|
|
|
def test_a_table_cell_escapes_pipes_and_tags_but_not_inside_code():
|
|
cell = reference_doc._cell('says "ingest <file>" or `a <b> | c` | d')
|
|
assert cell == 'says "ingest <file>" or `a <b> \\| c` \\| d'
|
|
|
|
|
|
@pytest.fixture
|
|
def exported_instance(tmp_path, monkeypatch):
|
|
"""A `dist export` of this tree, adopted and then changed the way an
|
|
instance may: a type-spec of its own making, and a skill of its own."""
|
|
target = tmp_path / "instance"
|
|
dist_cmd.run_export(target)
|
|
monkeypatch.setattr(config, "ROOT", target)
|
|
monkeypatch.setattr(resolver, "_repo_root", target)
|
|
for template in sorted((target / "types").glob("*.template")):
|
|
template.rename(template.with_suffix(""))
|
|
entity = target / "types" / "entity.md"
|
|
entity.write_text(
|
|
entity.read_text(encoding="utf-8").replace(
|
|
"description: Base type for entity pages", "description: This instance's entities"
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
own = target / "instructions" / "own-skill"
|
|
own.mkdir()
|
|
(own / "SKILL.md").write_text(
|
|
"---\nname: own-skill\ndescription: Does this instance's thing. Use when asked.\n---\n# Own\n",
|
|
encoding="utf-8",
|
|
)
|
|
return target
|
|
|
|
|
|
def test_an_adapted_instance_leaves_both_pages_alone(exported_instance):
|
|
root = exported_instance
|
|
assert version_mod.stamp_file().is_file()
|
|
pages = {
|
|
relative: (root / relative).read_text(encoding="utf-8")
|
|
for relative in (reference_doc.SKILLS, reference_doc.PAGE_TYPES)
|
|
}
|
|
assert docs_verify.check_reference_regions() == []
|
|
|
|
result = runner.invoke(app, ["docs", "reference", "--apply"])
|
|
assert result.exit_code == 0, result.output
|
|
assert "Left alone in this instance" in result.output
|
|
for relative, before in pages.items():
|
|
assert (root / relative).read_text(encoding="utf-8") == before, relative
|
|
assert docs_verify.check_reference_regions() == []
|
|
|
|
|
|
# --- a fixture tree -------------------------------------------------------
|
|
|
|
GROUPS = (("Fixture things", ("frobnicate",)), ("Other", ("gate keep",)))
|
|
|
|
|
|
def _record(path: str, gates: tuple[str, ...] = (), summary: str = "Does a thing.") -> cli_contract.CommandRecord:
|
|
failures = [cli_contract.Failure(cause="It broke", reaction="Show the user and stop")]
|
|
if gates:
|
|
failures.append(cli_contract.Failure(cause="Needs a person", reaction="Wait", code=42))
|
|
return cli_contract.CommandRecord(
|
|
path=path,
|
|
summary=summary,
|
|
synopsis=(cli_contract.Variant(usage=f"{path} [--flag]"),),
|
|
properties=cli_contract.Properties(
|
|
effect=cli_contract.Effect.WRITE,
|
|
idempotent=cli_contract.Idempotent.NO,
|
|
atomic="Yes",
|
|
budget=cli_contract.Budget.EXEMPT_WITHOUT_ARGS,
|
|
gates=gates,
|
|
),
|
|
notes=("Does a thing, mechanically.",),
|
|
failures=tuple(failures),
|
|
examples=("tools/wikitool frobnicate --flag",),
|
|
never=("Never frobnicate twice.",),
|
|
see_also=("`wikitool other` - a neighbour",),
|
|
)
|
|
|
|
|
|
HEADS = {
|
|
"docs/reference/commands/fixture-things.md": "# Fixture things\n\nThings, by hand.\n",
|
|
"docs/reference/commands/other.md": "# Other\n\nThe other one.\n",
|
|
"docs/reference/commands/README.md": "# Command reference\n\nAll of it.\n",
|
|
"docs/reference/exit-codes.md": "# Exit codes and gates\n\nWhere it stops.\n",
|
|
"docs/reference/skills.md": "# Skill catalogue\n\nThe skills.\n",
|
|
"docs/reference/page-types.md": "# Page types\n\nThe types.\n",
|
|
}
|
|
REGION_OF = {
|
|
"docs/reference/commands/fixture-things.md": reference_doc.COMMANDS_REGION,
|
|
"docs/reference/commands/other.md": reference_doc.COMMANDS_REGION,
|
|
"docs/reference/commands/README.md": reference_doc.INDEX_REGION,
|
|
"docs/reference/exit-codes.md": reference_doc.EXIT_CODES_REGION,
|
|
"docs/reference/skills.md": reference_doc.SKILLS_REGION,
|
|
"docs/reference/page-types.md": reference_doc.PAGE_TYPES_REGION,
|
|
}
|
|
|
|
|
|
def _page_type(description: str = "A fixture type.") -> reference_doc.PageType:
|
|
return reference_doc.PageType(
|
|
described={
|
|
"name": "gadget", "description": description, "subtype_field": "gadget_type",
|
|
"root": "kb", "base_dir": "gadgets", "title_prefix": None,
|
|
"fields": [
|
|
{"field": "type", "required": True, "type": "string", "enum": None},
|
|
{"field": "gadget_type", "required": True, "type": "string", "enum": ["widget"]},
|
|
{"field": "tags", "required": False, "type": "array", "enum": None},
|
|
],
|
|
},
|
|
subtypes=(reference_doc.Subtype("widget", "widgets", "Widgets"),),
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def tree(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(config, "ROOT", tmp_path)
|
|
records = {"frobnicate": _record("frobnicate"), "gate keep": _record("gate keep", ("keep",))}
|
|
monkeypatch.setattr(cli_contract, "all_records", lambda: dict(records))
|
|
monkeypatch.setattr(cli_contract, "GROUPS", GROUPS)
|
|
monkeypatch.setattr(cli, "options_texts", lambda paths: {p: " --flag A flag." for p in paths})
|
|
monkeypatch.setattr(
|
|
reference_doc, "shipped_skills",
|
|
lambda: [reference_doc.Skill("fixture-skill", "Does a thing. Use when asked.")],
|
|
)
|
|
monkeypatch.setattr(reference_doc, "page_types", lambda: [_page_type()])
|
|
for relative, head in HEADS.items():
|
|
path = tmp_path / relative
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
name = REGION_OF[relative]
|
|
path.write_text(
|
|
f"{head}\n{blocks.open_marker(name)}\n{blocks.close_marker(name)}\n", encoding="utf-8"
|
|
)
|
|
return tmp_path, records
|
|
|
|
|
|
def _snapshot(root) -> dict[str, str]:
|
|
return {
|
|
p.relative_to(root).as_posix(): p.read_text(encoding="utf-8")
|
|
for p in sorted(root.rglob("*.md"))
|
|
}
|
|
|
|
|
|
def test_a_fresh_tree_is_reported_stale_until_applied(tree):
|
|
root, _ = tree
|
|
issues = docs_verify.check_reference_regions()
|
|
assert len(issues) == len(HEADS)
|
|
assert all("stale" in issue and "docs reference --apply" in issue for issue in issues)
|
|
|
|
dry = runner.invoke(app, ["docs", "reference"])
|
|
assert dry.exit_code == 0, dry.output
|
|
assert "would change" in dry.output
|
|
assert docs_verify.check_reference_regions() != []
|
|
|
|
result = runner.invoke(app, ["docs", "reference", "--apply"])
|
|
assert result.exit_code == 0, result.output
|
|
assert docs_verify.check_reference_regions() == []
|
|
|
|
|
|
def test_apply_twice_changes_nothing_the_second_time(tree):
|
|
root, _ = tree
|
|
runner.invoke(app, ["docs", "reference", "--apply"])
|
|
first = _snapshot(root)
|
|
second_run = runner.invoke(app, ["docs", "reference", "--apply"])
|
|
assert second_run.exit_code == 0
|
|
assert "already current" in second_run.output
|
|
assert _snapshot(root) == first
|
|
|
|
|
|
def test_the_head_survives_apply_byte_for_byte(tree):
|
|
root, _ = tree
|
|
runner.invoke(app, ["docs", "reference", "--apply"])
|
|
for relative, head in HEADS.items():
|
|
text = (root / relative).read_text(encoding="utf-8")
|
|
assert text.startswith(head), relative
|
|
|
|
|
|
def test_a_changed_record_makes_verify_name_the_page_and_the_command(tree, monkeypatch):
|
|
root, records = tree
|
|
runner.invoke(app, ["docs", "reference", "--apply"])
|
|
records["frobnicate"] = _record("frobnicate", summary="Does a different thing.")
|
|
monkeypatch.setattr(cli_contract, "all_records", lambda: dict(records))
|
|
issues = docs_verify.check_reference_regions()
|
|
assert any(
|
|
"docs/reference/commands/fixture-things.md" in issue and "docs reference --apply" in issue
|
|
for issue in issues
|
|
)
|
|
|
|
|
|
def test_a_generated_page_leaves_out_reactions_and_never(tree):
|
|
root, _ = tree
|
|
runner.invoke(app, ["docs", "reference", "--apply"])
|
|
text = (root / "docs/reference/commands/fixture-things.md").read_text(encoding="utf-8")
|
|
assert "It broke" in text
|
|
assert "Show the user and stop" not in text
|
|
assert "Never frobnicate twice." not in text
|
|
assert "- Counts toward the session call budget: only when called with arguments" in text
|
|
assert "```text\n --flag A flag.\n```" in text
|
|
|
|
|
|
def test_a_long_page_gets_its_toc_outside_the_generated_region(tree, monkeypatch):
|
|
"""The first `##` of a group page is generated; a TOC inserted before it
|
|
would sit inside the region and be wiped by the next regeneration."""
|
|
root, records = tree
|
|
many = tuple(f"cmd{i}" for i in range(12))
|
|
for path in many:
|
|
records[path] = _record(path)
|
|
monkeypatch.setattr(cli_contract, "all_records", lambda: dict(records))
|
|
monkeypatch.setattr(cli_contract, "GROUPS", (("Fixture things", many), GROUPS[1]))
|
|
assert runner.invoke(app, ["docs", "reference", "--apply"]).exit_code == 0
|
|
page = root / "docs/reference/commands/fixture-things.md"
|
|
text = page.read_text(encoding="utf-8")
|
|
assert toc.needs_toc(text)
|
|
assert text.index(blocks.close_marker(toc.REGION_NAME)) < text.index(
|
|
blocks.open_marker(reference_doc.COMMANDS_REGION)
|
|
)
|
|
assert toc.upsert(text) == text
|
|
assert docs_verify.check_reference_regions() == []
|
|
|
|
|
|
def test_a_group_without_a_page_fails_and_nothing_is_created(tree, monkeypatch):
|
|
root, _ = tree
|
|
monkeypatch.setattr(cli_contract, "GROUPS", GROUPS + (("New group", ()),))
|
|
issues = docs_verify.check_reference_regions()
|
|
assert any("docs/reference/commands/new-group.md is missing" in issue for issue in issues)
|
|
result = runner.invoke(app, ["docs", "reference", "--apply"])
|
|
assert result.exit_code == 1
|
|
assert "docs/reference/commands/new-group.md" in result.output
|
|
assert not (root / "docs/reference/commands/new-group.md").exists()
|
|
|
|
|
|
def test_a_page_without_its_marker_pair_fails_and_is_left_alone(tree):
|
|
root, _ = tree
|
|
page = root / "docs/reference/exit-codes.md"
|
|
page.write_text("# Exit codes and gates\n\nNo region here.\n", encoding="utf-8")
|
|
issues = docs_verify.check_reference_regions()
|
|
assert any("exit-codes.md has no" in issue for issue in issues)
|
|
result = runner.invoke(app, ["docs", "reference", "--apply"])
|
|
assert result.exit_code == 1
|
|
assert page.read_text(encoding="utf-8") == "# Exit codes and gates\n\nNo region here.\n"
|
|
|
|
|
|
def test_a_page_without_a_group_fails(tree):
|
|
root, _ = tree
|
|
(root / "docs/reference/commands/gone.md").write_text("# Gone\n", encoding="utf-8")
|
|
issues = docs_verify.check_reference_regions()
|
|
assert any("gone.md belongs to no group" in issue for issue in issues)
|
|
|
|
|
|
def test_a_tree_without_docs_passes(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(config, "ROOT", tmp_path)
|
|
assert docs_verify.check_reference_regions() == []
|
|
|
|
|
|
def test_exit_codes_lists_each_gate_and_its_exit_42_causes(tree):
|
|
root, _ = tree
|
|
runner.invoke(app, ["docs", "reference", "--apply"])
|
|
page = (root / "docs/reference/exit-codes.md").read_text(encoding="utf-8")
|
|
assert "### keep\n\n- [`gate keep`](commands/other.md#gate-keep)" in page
|
|
assert "- [`gate keep`](commands/other.md#gate-keep): Needs a person" in page
|
|
assert "frobnicate" not in page
|
|
|
|
|
|
def test_the_index_links_every_command_to_its_section(tree):
|
|
root, _ = tree
|
|
runner.invoke(app, ["docs", "reference", "--apply"])
|
|
index = (root / "docs/reference/commands/README.md").read_text(encoding="utf-8")
|
|
assert "- [`frobnicate`](fixture-things.md#frobnicate) - Does a thing." in index
|
|
assert "- [`gate keep`](other.md#gate-keep) - Does a thing." in index
|
|
|
|
|
|
def test_a_changed_skill_or_type_makes_verify_name_its_page(tree, monkeypatch):
|
|
runner.invoke(app, ["docs", "reference", "--apply"])
|
|
assert docs_verify.check_reference_regions() == []
|
|
|
|
monkeypatch.setattr(
|
|
reference_doc, "shipped_skills",
|
|
lambda: [reference_doc.Skill("fixture-skill", "Does another thing. Use when asked.")],
|
|
)
|
|
monkeypatch.setattr(reference_doc, "page_types", lambda: [_page_type("A changed type.")])
|
|
issues = docs_verify.check_reference_regions()
|
|
for page in ("docs/reference/skills.md", "docs/reference/page-types.md"):
|
|
assert any(page in issue and "docs reference --apply" in issue for issue in issues), page
|
|
|
|
|
|
def test_the_fixture_skill_and_type_render(tree):
|
|
root, _ = tree
|
|
runner.invoke(app, ["docs", "reference", "--apply"])
|
|
skills = (root / "docs/reference/skills.md").read_text(encoding="utf-8")
|
|
assert "| `fixture-skill` | Does a thing. | asked. |" in skills
|
|
types = (root / "docs/reference/page-types.md").read_text(encoding="utf-8")
|
|
assert "- Written to: `kb/gadgets/<subtype directory>/`" in types
|
|
assert "| `widget` | `kb/gadgets/widgets/` | Widgets |" in types
|
|
assert "- `gadget_type` - string, one of `widget`" in types
|
|
assert "`tags`" not in types
|
|
|
|
|
|
def test_an_instance_skips_a_missing_skill_or_type_page(tree):
|
|
"""In an instance the two pages are not the tree's to check - not even
|
|
whether they exist."""
|
|
root, _ = tree
|
|
runner.invoke(app, ["docs", "reference", "--apply"])
|
|
(root / version_mod.RELEASE_STAMP_FILENAME).write_text("{}", encoding="utf-8")
|
|
(root / "docs/reference/skills.md").unlink()
|
|
(root / "docs/reference/page-types.md").write_text("# Page types\n", encoding="utf-8")
|
|
assert docs_verify.check_reference_regions() == []
|
|
result = runner.invoke(app, ["docs", "reference", "--apply"])
|
|
assert result.exit_code == 0, result.output
|
|
assert "Left alone in this instance: docs/reference/skills.md, docs/reference/page-types.md" in result.output
|
|
assert not (root / "docs/reference/skills.md").exists()
|