feat: export guidelines - the guideline pages as a generated GUIDELINES.md, pushed into the captured repositories behind the Guideline Push Gate (#179)
Files changed: - AGENTS.md - CHANGES.md - README.md - VERSION - docs/why-gates-are-code.md - instructions/gates.md - instructions/kb-profiles.md - kb/CONTRACT.md - kb/CONVENTIONS.md - kb/CONVENTIONS.md.template - raw/CONTRACT.md - tools/CONTRACT.md - tools/README.md - tools/chemenu/cli.py - tools/chemenu/cli_contract.py - tools/chemenu/commands/export_cmd.py - tools/chemenu/commands/page_ops.py - tools/chemenu/commands/raw_cmd.py - tools/chemenu/commands/search.py - tools/chemenu/guideline_export.py - tools/chemenu/kb_scan.py - tools/chemenu/repo_capture.py - tools/chemenu/search/filters.py - tools/chemenu/tests/test_cli.py - tools/chemenu/tests/test_export_guidelines.py Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
This commit is contained in:
torbenandClaude Opus 5.5 committed
1 parent
ccede04b97
commit
283cdae8be
25 files changed
+1753
-68
No files matched your search
@@ -0,0 +1,429 @@
|
||||
"""`wikitool export guidelines` - the instance's guideline pages as one
|
||||
generated `GUIDELINES.md`, printed, or written into the captured repositories
|
||||
behind the Guideline Push Gate.
|
||||
|
||||
The core - selection, rendering, the header - is `chemenu/guideline_export.py`;
|
||||
the git write is `repo_capture.commit_file`/`push_commit`. This module decides
|
||||
which captured repositories are targets, which of them take part, and when the
|
||||
gate opens.
|
||||
|
||||
**Targets are the captured repositories, and nothing else.** The `repo` and
|
||||
`ref` of every `_capture.json` under `raw/` - the one declaration that this
|
||||
instance follows a repository, now read in both directions. A second list of
|
||||
targets would be a second declaration of the same thing to drift.
|
||||
|
||||
**A repository takes part by carrying the file.** Only a branch whose root
|
||||
`GUIDELINES.md` already starts with the guideline export header is written to;
|
||||
a hand-written `GUIDELINES.md` is never replaced, and one whose header names a
|
||||
different `instance=` belongs to that instance. A bare header line is the
|
||||
opt-in.
|
||||
|
||||
**Why a gate, and why this shape.** A push into another repository is the most
|
||||
irreversible thing the stack does, and a public target publishes the content -
|
||||
the `instance=` URL included - the moment it lands. So it is held to the same
|
||||
token form as the Mass-Update and Upload Review gates: the first run fetches,
|
||||
checks and builds everything, pushes nothing and exits 42 with every target's
|
||||
status and diff; the token digests exactly that set (URL, branch, old tip, new
|
||||
blob), so a moved branch, a changed guideline or a new target asks again. The
|
||||
Publish-Remote Gate is deliberately not applied: the targets are declared by
|
||||
committed manifests already, and listing the same URLs in
|
||||
`.wikitool-remotes.json` would be the second declaration again.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
import hashlib
|
||||
import json
|
||||
import shlex
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from rich.markup import escape
|
||||
|
||||
from chemenu import cli_contract, config, guideline_export, repo_capture
|
||||
from chemenu.commands._util import EXIT_NEEDS_CLEARANCE, console, fail, rel_path
|
||||
from chemenu.errors import BackendError, ValidationError
|
||||
from chemenu.search import filters
|
||||
|
||||
app = typer.Typer(help="Generate files from the wiki for other repositories.")
|
||||
|
||||
GATE = "guideline-push"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Target:
|
||||
"""One `(repo, ref)` pair - several bundles captured from one branch are
|
||||
one target. `error` is set when a manifest could not be read."""
|
||||
repo: str
|
||||
ref: str
|
||||
bundles: list[str] = field(default_factory=list)
|
||||
error: str = ""
|
||||
|
||||
@property
|
||||
def label(self) -> str:
|
||||
return f"{self.repo} {self.ref}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Plan:
|
||||
target: Target
|
||||
status: str # new | changed | unchanged | skipped
|
||||
reason: str = ""
|
||||
tip: str = ""
|
||||
blob: str = ""
|
||||
old: bytes = b""
|
||||
|
||||
@property
|
||||
def writes(self) -> bool:
|
||||
return self.status in ("new", "changed")
|
||||
|
||||
|
||||
def _targets(only: list[Path]) -> list[Target]:
|
||||
manifests = repo_capture.captured_manifests(config.RAW_DIR)
|
||||
if only:
|
||||
wanted = {p.resolve() for p in only}
|
||||
known = {m.parent.resolve() for m in manifests}
|
||||
unknown = sorted(rel_path(p) for p in only if p.resolve() not in known)
|
||||
if unknown:
|
||||
raise ValidationError(
|
||||
f"--bundle {', '.join(unknown)}: not a captured bundle under raw/ (a directory "
|
||||
f"holding {repo_capture.MANIFEST_NAME})."
|
||||
)
|
||||
manifests = [m for m in manifests if m.parent.resolve() in wanted]
|
||||
targets: dict[tuple[str, str], Target] = {}
|
||||
broken: list[Target] = []
|
||||
for path in manifests:
|
||||
bundle = rel_path(path.parent)
|
||||
try:
|
||||
manifest = repo_capture.read_manifest(path)
|
||||
except ValidationError as exc:
|
||||
broken.append(Target(bundle, "?", [bundle], f"manifest unreadable: {exc}"))
|
||||
continue
|
||||
target = targets.setdefault((manifest.repo, manifest.ref), Target(manifest.repo, manifest.ref))
|
||||
target.bundles.append(bundle)
|
||||
return [*targets.values(), *broken]
|
||||
|
||||
|
||||
def _plan(target: Target, export: guideline_export.Export) -> Plan:
|
||||
"""Where this target stands - fetched, never written to."""
|
||||
if target.error:
|
||||
return Plan(target, "skipped", target.error)
|
||||
if repo_capture.is_tag_pattern(target.ref):
|
||||
return Plan(target, "skipped", "captured by a tag pattern - a tag takes no commit")
|
||||
try:
|
||||
with repo_capture.cache_repo(target.repo) as repo:
|
||||
resolved = repo_capture.resolve_ref(target.repo, target.ref, repo)
|
||||
tip = repo_capture.fetch(target.repo, resolved.refname, repo)
|
||||
existing = repo_capture.read_path(repo, tip, guideline_export.TARGET_PATH)
|
||||
blob = repo_capture.blob_id(repo, export.data)
|
||||
except BackendError as exc:
|
||||
return Plan(target, "skipped", f"not reachable: {exc}")
|
||||
except ValidationError as exc: # the branch does not exist
|
||||
return Plan(target, "skipped", str(exc))
|
||||
if existing is None:
|
||||
return Plan(target, "skipped", f"no {guideline_export.TARGET_PATH} - not opted in", tip)
|
||||
fields = guideline_export.guidelines_header(existing)
|
||||
if fields is None:
|
||||
return Plan(
|
||||
target, "skipped",
|
||||
f"{guideline_export.TARGET_PATH} has no export header - hand-written, never replaced", tip,
|
||||
)
|
||||
owner = fields.get("instance")
|
||||
if owner is not None and owner != export.instance:
|
||||
return Plan(target, "skipped", f"delivered by another instance ({owner})", tip)
|
||||
if existing == export.data:
|
||||
return Plan(target, "unchanged", tip=tip)
|
||||
return Plan(target, "changed" if owner else "new", tip=tip, blob=blob, old=existing)
|
||||
|
||||
|
||||
def confirm_token(plans: list[Plan]) -> str:
|
||||
"""sha256 over every target the run would write - URL, branch, the tip it
|
||||
builds on and the blob it writes - cut to 12 hex chars, the shape of
|
||||
`git_publish.changeset_token`. The new commit's own id is not part of it:
|
||||
`commit-tree` stamps the time, so it differs on every run."""
|
||||
payload = json.dumps(
|
||||
sorted([p.target.repo, p.target.ref, p.tip, p.blob] for p in plans if p.writes),
|
||||
sort_keys=True,
|
||||
)
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:12]
|
||||
|
||||
|
||||
def _rerun(token: str, predicates: list[str], bundles: list[Path]) -> str:
|
||||
parts = ["tools/wikitool", "export", "guidelines", "--confirm", token, "--push"]
|
||||
for raw in predicates:
|
||||
parts += ["--field", raw]
|
||||
for bundle in bundles:
|
||||
parts += ["--bundle", rel_path(bundle)]
|
||||
return " ".join(shlex.quote(part) for part in parts)
|
||||
|
||||
|
||||
def _diff(plan: Plan, export: guideline_export.Export) -> str:
|
||||
old = plan.old.decode("utf-8", "replace").splitlines(keepends=True)
|
||||
new = export.text.splitlines(keepends=True)
|
||||
name = guideline_export.TARGET_PATH
|
||||
return "".join(difflib.unified_diff(old, new, f"a/{name}", f"b/{name}"))
|
||||
|
||||
|
||||
def _status_line(plan: Plan) -> str:
|
||||
detail = f" ({plan.reason})" if plan.reason else ""
|
||||
return f" {plan.target.label}: {plan.status}{detail}"
|
||||
|
||||
|
||||
def _clearance(plans: list[Plan], export, token: str, rerun: str, stale: Optional[str]) -> str:
|
||||
writes = [p for p in plans if p.writes]
|
||||
lines = [
|
||||
f"This run would push one commit changing only {guideline_export.TARGET_PATH} into "
|
||||
f"{len(writes)} repositor{'y' if len(writes) == 1 else 'ies'}. A push there is visible at "
|
||||
"once - publicly, for a public repository - and not cheaply reversible. Nothing was pushed.",
|
||||
"",
|
||||
]
|
||||
if stale:
|
||||
lines += [
|
||||
f"The token you passed ({stale}) does not match this set - a branch moved, a guideline "
|
||||
"changed, a target came or went, or the token was invented. Here is the current state.",
|
||||
"",
|
||||
]
|
||||
lines += [
|
||||
"THE USER CANNOT SEE THIS OUTPUT. It went to your context, not to their screen.",
|
||||
"Reproduce every target line and every diff below in your reply, and stop there. Run no "
|
||||
"further commands in this turn.",
|
||||
"",
|
||||
f"TARGETS ({len(plans)}) - reproduce this in your reply:",
|
||||
"",
|
||||
*(_status_line(p) for p in plans),
|
||||
]
|
||||
for plan in writes:
|
||||
lines += ["", f"DIFF {plan.target.label}:", "", _diff(plan, export).rstrip("\n")]
|
||||
lines += [
|
||||
"",
|
||||
"Once they have replied approving exactly this, the line that pushes it is:",
|
||||
"",
|
||||
f" {rerun}",
|
||||
"",
|
||||
f"The token {token} covers each target's branch tip and the exact file; if either moves, "
|
||||
"clearance is asked again.",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _write(plan: Plan, export: guideline_export.Export, identity: tuple[str, str]) -> tuple[str, bool]:
|
||||
"""Commit and push one target. Returns its result line and whether it
|
||||
counts as a failure (rejected, or a git error)."""
|
||||
message = f"Update {guideline_export.TARGET_PATH} from {export.instance} at {export.commit[:12]}"
|
||||
label = plan.target.label
|
||||
try:
|
||||
with repo_capture.cache_repo(plan.target.repo) as repo:
|
||||
commit = repo_capture.commit_file(
|
||||
repo, plan.tip, guideline_export.TARGET_PATH, export.data, message, identity
|
||||
)
|
||||
result = repo_capture.push_commit(plan.target.repo, repo, commit, plan.target.ref)
|
||||
except BackendError as exc:
|
||||
return f" {label}: failed ({exc})", True
|
||||
if not result.accepted:
|
||||
return f" {label}: rejected ({result.detail}) - the branch moved; run again", True
|
||||
return f" {label}: written {plan.tip[:12]} -> {commit[:12]}", False
|
||||
|
||||
|
||||
@app.command("guidelines")
|
||||
@cli_contract.record(cli_contract.CommandRecord(
|
||||
path="export guidelines",
|
||||
summary="Render the guideline pages into one generated `GUIDELINES.md`, or push it into the "
|
||||
"captured repositories that opted in (**Guideline Push Gate**).",
|
||||
synopsis=(
|
||||
cli_contract.Variant(
|
||||
usage="export guidelines [--field <predicate> ...] [--kind/--subtype/--collection/"
|
||||
"--tag <v>]",
|
||||
notes="Prints the file to stdout; no network.",
|
||||
),
|
||||
cli_contract.Variant(
|
||||
usage="export guidelines <predicates> --push [--bundle <raw-bundle> ...] "
|
||||
"[--confirm TOKEN]",
|
||||
notes="Writes it into the captured repositories, behind the gate.",
|
||||
),
|
||||
),
|
||||
properties=cli_contract.Properties(
|
||||
effect=cli_contract.Effect.WRITE,
|
||||
idempotent=cli_contract.Idempotent.YES,
|
||||
atomic="Per target - one commit pushed without force, or nothing; targets already "
|
||||
"written stay written when a later one fails",
|
||||
budget=cli_contract.Budget.COUNTED,
|
||||
network=cli_contract.Network.YES,
|
||||
gates=(GATE,),
|
||||
),
|
||||
notes=(
|
||||
"Selects pages with `search`'s own predicates and no text: `--field` (repeatable, AND), "
|
||||
"`--kind`, `--subtype`, `--collection`, `--tag`. Which filter selects this instance's "
|
||||
"guidelines is written down in `kb/CONVENTIONS.md`.",
|
||||
"Renders mechanically: line 1 is `<!-- wikitool:export kind=guidelines instance=<origin "
|
||||
"URL without userinfo, or local> commit=<newest commit touching a selected page> - "
|
||||
"generated, do not edit by hand -->`; then each page sorted by title, as `# <title>` and "
|
||||
"its text without frontmatter, the generated links and footnote blocks, citation markers "
|
||||
"(`[^id]`, legacy `^[[...]]`), with `[[Title|Text]]` as `Text` and `[[Title]]` as "
|
||||
"`Title`. A leading H1 of the page is replaced by the title heading. Code is never "
|
||||
"rewritten. LF endings, no timestamp: the same pages give the same bytes.",
|
||||
"Refuses before printing or pushing anything: no predicate, no match, an unknown field, "
|
||||
"any page under `kb/` with unreadable frontmatter, uncommitted changes in `kb/` or "
|
||||
"`types/` (untracked files included), or no git checkout.",
|
||||
"`--push` targets every captured repository - each `(repo, ref)` from a `_capture.json` "
|
||||
"under `raw/`, `--bundle` narrowing it. A target is skipped with its reason when its ref "
|
||||
"rule is a tag pattern, it cannot be reached or has no such branch, its root has no "
|
||||
"`GUIDELINES.md` (not opted in), its `GUIDELINES.md` has no export header (hand-written), "
|
||||
"or the header names another `instance=`. A header line alone is the opt-in.",
|
||||
"Guideline Push Gate: `--push` without a matching `--confirm` fetches and builds "
|
||||
"everything, pushes nothing, and exits 42 with each target's status (`new`, `changed`, "
|
||||
"`unchanged`, `skipped`), the diff for every target it would write, and the re-run line. "
|
||||
"The token digests URL, branch, old tip and new blob of every target to write.",
|
||||
"With a matching token, each target gets exactly one commit on the fetched tip changing "
|
||||
"only `GUIDELINES.md`, authored as the instance checkout's `user.name`/`user.email`, "
|
||||
"pushed without force; one line per target: `unchanged`, `written <old> -> <new>`, "
|
||||
"`skipped (<reason>)`, `rejected (<reason>)` or `failed (<reason>)`.",
|
||||
"Nothing to write (every target unchanged or skipped) ends with exit 0 and no gate. A "
|
||||
"second run after a successful one finds every target unchanged.",
|
||||
"The target repository's own `AGENTS.md` points at the file (Claude Code: "
|
||||
"`@GUIDELINES.md`); that belongs to the repository, not to this command.",
|
||||
),
|
||||
failures=(
|
||||
cli_contract.Failure(
|
||||
cause="No predicate, no page matches, a malformed predicate or an unknown field",
|
||||
reaction="Use the filter `kb/CONVENTIONS.md` names for this instance's guidelines, and "
|
||||
"retry once",
|
||||
),
|
||||
cli_contract.Failure(
|
||||
cause="A page under `kb/` has unreadable frontmatter, `kb/` or `types/` has "
|
||||
"uncommitted changes, or the instance is not a git checkout",
|
||||
reaction="Fix the named pages, or publish the changes first, then retry once",
|
||||
),
|
||||
cli_contract.Failure(
|
||||
cause="`--push` with no `user.name` or `user.email` in the instance checkout's git "
|
||||
"configuration - checked before any target is fetched",
|
||||
reaction="Show the message to the user; the identity is theirs to set, never yours",
|
||||
),
|
||||
cli_contract.Failure(
|
||||
cause="`--bundle` names no captured bundle under `raw/`, or `--bundle`/`--confirm` "
|
||||
"was given without `--push`",
|
||||
reaction="Fix the argument and retry once",
|
||||
),
|
||||
cli_contract.Failure(
|
||||
cause="A target rejected the push (its branch moved since the fetch) or a git call "
|
||||
"failed - the other targets were still served",
|
||||
reaction="Show the lines to the user. A rejected target is served by running the "
|
||||
"whole command again, which asks for clearance again",
|
||||
),
|
||||
cli_contract.Failure(
|
||||
code=0,
|
||||
cause="A target was skipped - tag rule, not reachable, not opted in, hand-written "
|
||||
"file, another instance's file",
|
||||
reaction="",
|
||||
),
|
||||
cli_contract.Failure(
|
||||
cause="Guideline Push Gate: `--push` without `--confirm`, or with a token that does "
|
||||
"not match the current set of targets",
|
||||
reaction=cli_contract.token_gate_reaction("--confirm"),
|
||||
code=42,
|
||||
),
|
||||
),
|
||||
examples=(
|
||||
"tools/wikitool export guidelines --tag guideline",
|
||||
"tools/wikitool export guidelines --tag guideline --push",
|
||||
"tools/wikitool export guidelines --confirm <token> --push --field tags=guideline # "
|
||||
"re-run after exit 42, once the user approved every diff",
|
||||
),
|
||||
never=(
|
||||
"Never pass a `--confirm` token the user has not seen and approved.",
|
||||
"Never edit a `GUIDELINES.md` in a target repository by hand to make it match - the "
|
||||
"next export overwrites it; change the guideline page instead.",
|
||||
),
|
||||
see_also=(
|
||||
"`wikitool search` - the same predicates, to preview the selection",
|
||||
"`wikitool raw capture` - how a repository becomes a target",
|
||||
"`instructions/gates.md` - the gate procedure",
|
||||
),
|
||||
))
|
||||
def export_guidelines_command(
|
||||
field_: Optional[list[str]] = typer.Option(
|
||||
None, "--field", "-f",
|
||||
help="Frontmatter predicate, repeatable (AND) - the same forms as `search --field`.",
|
||||
),
|
||||
kind: Optional[str] = typer.Option(None, "--kind", help="Shorthand for --field kind=<value>."),
|
||||
subtype: Optional[str] = typer.Option(None, "--subtype", help="Shorthand for --field subtype=<value>."),
|
||||
collection: Optional[str] = typer.Option(
|
||||
None, "--collection", help="Shorthand for --field collection=<value>."
|
||||
),
|
||||
tag: Optional[str] = typer.Option(None, "--tag", help="Shorthand for --field tags=<value>."),
|
||||
push: bool = typer.Option(
|
||||
False, "--push", help="Write the file into the captured repositories (Guideline Push Gate)."
|
||||
),
|
||||
bundle: Optional[list[Path]] = typer.Option(
|
||||
None, "--bundle", help="Only the repository of this captured bundle under raw/; repeatable."
|
||||
),
|
||||
confirm: Optional[str] = typer.Option(
|
||||
None, "--confirm", help="The token from a prior exit 42, once a human has approved it."
|
||||
),
|
||||
):
|
||||
"""Render the guideline pages into GUIDELINES.md - printed, or pushed into
|
||||
the captured repositories that carry one."""
|
||||
bundles = [b if b.is_absolute() else config.ROOT / b for b in (bundle or [])]
|
||||
if not push and (bundles or confirm):
|
||||
fail("--bundle and --confirm only apply with --push.")
|
||||
raw = filters.raw_predicates(field_, kind, subtype, collection, tag)
|
||||
try:
|
||||
predicates = tuple(filters.parse_predicate(r) for r in raw)
|
||||
export = guideline_export.build(predicates)
|
||||
except ValidationError as exc:
|
||||
fail(escape(str(exc)))
|
||||
return
|
||||
|
||||
if not push:
|
||||
typer.echo(export.text, nl=False)
|
||||
return
|
||||
|
||||
identity = guideline_export.git_identity()
|
||||
if identity is None:
|
||||
fail(
|
||||
"No git identity in this instance checkout - `git config user.name` and `git config "
|
||||
"user.email` must both be set, since the commit in each target repository carries them. "
|
||||
"Nothing was fetched or pushed."
|
||||
)
|
||||
return
|
||||
try:
|
||||
targets = _targets(bundles)
|
||||
except ValidationError as exc:
|
||||
fail(escape(str(exc)))
|
||||
return
|
||||
if not targets:
|
||||
typer.echo("No captured repository under raw/ - nothing to write.")
|
||||
return
|
||||
|
||||
plans = [_plan(t, export) for t in targets]
|
||||
if not any(p.writes for p in plans):
|
||||
typer.echo(f"{len(plans)} target(s), nothing to write:")
|
||||
for plan in plans:
|
||||
typer.echo(_status_line(plan))
|
||||
return
|
||||
|
||||
token = confirm_token(plans)
|
||||
if confirm != token:
|
||||
console.print("[bold yellow]NEEDS USER CLEARANCE[/bold yellow] Guideline Push Gate")
|
||||
typer.echo(_clearance(plans, export, token, _rerun(token, raw, bundles), confirm))
|
||||
raise typer.Exit(code=EXIT_NEEDS_CLEARANCE)
|
||||
|
||||
failed = 0
|
||||
typer.echo(f"{len(plans)} target(s):")
|
||||
for plan in plans:
|
||||
if not plan.writes:
|
||||
typer.echo(_status_line(plan))
|
||||
continue
|
||||
line, bad = _write(plan, export, identity)
|
||||
failed += bad
|
||||
typer.echo(line)
|
||||
written = sum(1 for p in plans if p.writes) - failed
|
||||
typer.echo(f"{written} written, {failed} rejected or failed, {len(plans) - written - failed} "
|
||||
"unchanged or skipped.")
|
||||
if failed:
|
||||
console.print(
|
||||
f"[bold red]ERROR[/bold red] {failed} target(s) not written - what was written stays "
|
||||
"written; run the command again for the rest."
|
||||
)
|
||||
raise typer.Exit(code=1)
|
||||
Reference in new issue
Block a user