tools: command records, Telemetry group - examples, missing --fail-on-error exit line (#142)
Files changed: - CHANGES.md - VERSION - tools/CONTRACT.md - tools/chemenu/commands/eval_cmd.py
This commit is contained in:
1 parent
9617d722de
commit
71efdbe01b
4 files changed
+86
-20
No files matched your search
@@ -36,9 +36,20 @@ EVALS_DIR = config.REPORTS_DIR / "evals"
|
||||
atomic="Read-only",
|
||||
budget=cli_contract.Budget.EXEMPT,
|
||||
),
|
||||
notes="Most recent first. Read-only and exempt from the Iteration Budget Gate. Never fails; "
|
||||
"an empty list is a valid answer.",
|
||||
notes=(
|
||||
"Lists the sessions that have a trace under `reports/telemetry/`, most recent first.",
|
||||
"Never fails; an empty list is a valid answer.",
|
||||
"Read-only and exempt from the Iteration Budget Gate.",
|
||||
),
|
||||
failures=(),
|
||||
examples=(
|
||||
"tools/wikitool eval sessions",
|
||||
"tools/wikitool eval sessions --json",
|
||||
),
|
||||
see_also=(
|
||||
"`wikitool eval score` - scores one of them",
|
||||
"`EVALS.md` - how telemetry and evaluation work",
|
||||
),
|
||||
))
|
||||
def sessions_command(
|
||||
json_out: bool = typer.Option(False, "--json", help="Print the list as JSON"),
|
||||
@@ -71,19 +82,39 @@ def sessions_command(
|
||||
atomic="Read-only, apart from the files `--save`/`--markdown` write",
|
||||
budget=cli_contract.Budget.EXEMPT,
|
||||
),
|
||||
notes="Structural state from `lint`'s own checks (L1) plus trajectory rules over the trace "
|
||||
"(L2) - was a refused call repeated unchanged, was a gate flag passed without that gate "
|
||||
"having refused anything, did a publish of `kb/` pages go unlogged. Defaults to the current "
|
||||
"session. `--save` writes `reports/evals/<date>/<session>.{json,md}`. Read-only over `kb/` "
|
||||
"and exempt from the budget; see `EVALS.md`",
|
||||
failures=(cli_contract.Failure(
|
||||
label="",
|
||||
cause="No trace exists for the named session",
|
||||
reaction="Run `eval sessions` to see which ids exist. A session records nothing when "
|
||||
"telemetry is off - `WIKI_TRACE=0`, or a distributed instance with no opt-in "
|
||||
"(`wikitool doctor` says which) - so an absent trace is not necessarily a fault. Safe "
|
||||
"to retry",
|
||||
),),
|
||||
notes=(
|
||||
"Scores one traced session: structural state from `lint`'s own checks (L1) plus "
|
||||
"trajectory rules over the trace (L2) - was a refused call repeated unchanged, was a "
|
||||
"gate flag passed without that gate having refused anything, did a publish of `kb/` "
|
||||
"pages go unlogged.",
|
||||
"Defaults to the current session; `--session <id>` picks another.",
|
||||
"`--save` writes `reports/evals/<date>/<session>.{json,md}`; `--markdown` writes the "
|
||||
"report to the named file.",
|
||||
"A session records nothing when telemetry is off - `WIKI_TRACE=0`, or a distributed "
|
||||
"instance with no opt-in (`wikitool doctor` says which) - so an absent trace is not "
|
||||
"necessarily a fault.",
|
||||
"Read-only over `kb/`, safe to retry, and exempt from the Iteration Budget Gate.",
|
||||
),
|
||||
failures=(
|
||||
cli_contract.Failure(
|
||||
cause="No trace exists for the named session",
|
||||
reaction="Run `eval sessions` to see which ids exist; check with `doctor` whether "
|
||||
"telemetry is on",
|
||||
),
|
||||
cli_contract.Failure(
|
||||
cause="Only with `--fail-on-error`: the tree has hard errors or an invariant was "
|
||||
"violated",
|
||||
reaction="Act on the scorecard; re-run only to re-measure",
|
||||
),
|
||||
),
|
||||
examples=(
|
||||
"tools/wikitool eval score",
|
||||
"tools/wikitool eval score --session wiki-1727330000 --save",
|
||||
),
|
||||
see_also=(
|
||||
"`wikitool eval sessions` - which session ids exist",
|
||||
"`EVALS.md` - the scoring levels",
|
||||
),
|
||||
))
|
||||
def score_command(
|
||||
session: Optional[str] = typer.Option(
|
||||
|
||||
Reference in new issue
Block a user