tools: command records, Telemetry group - examples, missing --fail-on-error exit line (#142)
CI / verify (push) Successful in 1m14s
Release / release (push) Successful in 35s

Files changed:
- CHANGES.md
- VERSION
- tools/CONTRACT.md
- tools/chemenu/commands/eval_cmd.py
This commit is contained in:
torben committed 2026-09-26 09:24:15 +02:00
1 parent 9617d722de
commit 71efdbe01b
4 files changed
+86 -20

No files matched your search

+46 -15
View File
@@ -36,9 +36,20 @@ EVALS_DIR = config.REPORTS_DIR / "evals"
atomic="Read-only",
budget=cli_contract.Budget.EXEMPT,
),
notes="Most recent first. Read-only and exempt from the Iteration Budget Gate. Never fails; "
"an empty list is a valid answer.",
notes=(
"Lists the sessions that have a trace under `reports/telemetry/`, most recent first.",
"Never fails; an empty list is a valid answer.",
"Read-only and exempt from the Iteration Budget Gate.",
),
failures=(),
examples=(
"tools/wikitool eval sessions",
"tools/wikitool eval sessions --json",
),
see_also=(
"`wikitool eval score` - scores one of them",
"`EVALS.md` - how telemetry and evaluation work",
),
))
def sessions_command(
json_out: bool = typer.Option(False, "--json", help="Print the list as JSON"),
@@ -71,19 +82,39 @@ def sessions_command(
atomic="Read-only, apart from the files `--save`/`--markdown` write",
budget=cli_contract.Budget.EXEMPT,
),
notes="Structural state from `lint`'s own checks (L1) plus trajectory rules over the trace "
"(L2) - was a refused call repeated unchanged, was a gate flag passed without that gate "
"having refused anything, did a publish of `kb/` pages go unlogged. Defaults to the current "
"session. `--save` writes `reports/evals/<date>/<session>.{json,md}`. Read-only over `kb/` "
"and exempt from the budget; see `EVALS.md`",
failures=(cli_contract.Failure(
label="",
cause="No trace exists for the named session",
reaction="Run `eval sessions` to see which ids exist. A session records nothing when "
"telemetry is off - `WIKI_TRACE=0`, or a distributed instance with no opt-in "
"(`wikitool doctor` says which) - so an absent trace is not necessarily a fault. Safe "
"to retry",
),),
notes=(
"Scores one traced session: structural state from `lint`'s own checks (L1) plus "
"trajectory rules over the trace (L2) - was a refused call repeated unchanged, was a "
"gate flag passed without that gate having refused anything, did a publish of `kb/` "
"pages go unlogged.",
"Defaults to the current session; `--session <id>` picks another.",
"`--save` writes `reports/evals/<date>/<session>.{json,md}`; `--markdown` writes the "
"report to the named file.",
"A session records nothing when telemetry is off - `WIKI_TRACE=0`, or a distributed "
"instance with no opt-in (`wikitool doctor` says which) - so an absent trace is not "
"necessarily a fault.",
"Read-only over `kb/`, safe to retry, and exempt from the Iteration Budget Gate.",
),
failures=(
cli_contract.Failure(
cause="No trace exists for the named session",
reaction="Run `eval sessions` to see which ids exist; check with `doctor` whether "
"telemetry is on",
),
cli_contract.Failure(
cause="Only with `--fail-on-error`: the tree has hard errors or an invariant was "
"violated",
reaction="Act on the scorecard; re-run only to re-measure",
),
),
examples=(
"tools/wikitool eval score",
"tools/wikitool eval score --session wiki-1727330000 --save",
),
see_also=(
"`wikitool eval sessions` - which session ids exist",
"`EVALS.md` - the scoring levels",
),
))
def score_command(
session: Optional[str] = typer.Option(