From 82a22eaa93c36ad26707575be7df585a69a0c265 Mon Sep 17 00:00:00 2001 From: Torben Nehmer Date: Thu, 10 Sep 2026 23:38:22 +0200 Subject: [PATCH] stack: Telemetrie-Default nach Installationsform, Byte-Deckel und Session-Retention (schliesst #55) Files changed: - .gitea/workflows/ci.yml - .gitignore - CHANGES.md - EVALS.md - INSTALL-MCP.md - INSTALL.md - VERSION - instructions/setup-instance.md - reports/CONTRACT.md - tools/CONTRACT.md - tools/chemenu/commands/doctor.py - tools/chemenu/config.py - tools/chemenu/mcp/server.py - tools/chemenu/telemetry/policy.py - tools/chemenu/telemetry/schema.py - tools/chemenu/telemetry/writer.py - tools/chemenu/tests/conftest.py - tools/chemenu/tests/test_doctor.py - tools/chemenu/tests/test_mcp_server.py - tools/chemenu/tests/test_telemetry_emit.py - tools/chemenu/tests/test_telemetry_policy.py - tools/chemenu/version.py --- .gitea/workflows/ci.yml | 8 + .gitignore | 8 + CHANGES.md | 54 ++++++- EVALS.md | 55 ++++++- INSTALL-MCP.md | 15 +- INSTALL.md | 23 ++- VERSION | 2 +- instructions/setup-instance.md | 41 ++++- reports/CONTRACT.md | 20 ++- tools/CONTRACT.md | 4 +- tools/chemenu/commands/doctor.py | 35 ++++- tools/chemenu/config.py | 18 +++ tools/chemenu/mcp/server.py | 22 +-- tools/chemenu/telemetry/policy.py | 155 +++++++++++++++++++ tools/chemenu/telemetry/schema.py | 1 + tools/chemenu/telemetry/writer.py | 100 +++++++++++- tools/chemenu/tests/conftest.py | 16 +- tools/chemenu/tests/test_doctor.py | 39 +++++ tools/chemenu/tests/test_mcp_server.py | 19 +++ tools/chemenu/tests/test_telemetry_emit.py | 80 ++++++++++ tools/chemenu/tests/test_telemetry_policy.py | 127 +++++++++++++++ tools/chemenu/version.py | 3 +- 22 files changed, 801 insertions(+), 44 deletions(-) create mode 100644 tools/chemenu/telemetry/policy.py create mode 100644 tools/chemenu/tests/test_telemetry_policy.py diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 51e446d..9e36e78 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -250,3 +250,11 @@ jobs: # A fresh instance owes no migration: dist export declares its content # version, so `status` must answer rather than ask for a baseline. tools/wikitool migrate status + # Telemetry defaults off for a distributed instance (the + # .wikitool-release.json this export carries), so nothing above + # should have created a trace tree at all - see EVALS.md § "Whether + # it runs at all". + if [ -e reports/telemetry ]; then + echo "reports/telemetry/ exists in a fresh distributed instance - telemetry should default off" + exit 1 + fi diff --git a/.gitignore b/.gitignore index dd6c109..312cca4 100644 --- a/.gitignore +++ b/.gitignore @@ -117,6 +117,14 @@ npm-debug.log* # means unrestricted; `doctor` reports which. /.wikitool-remotes.json +# Telemetry opt-in/opt-out plus its two quantity caps (see EVALS.md and +# tools/chemenu/telemetry/policy.py). Per-checkout for the same reason as the +# allowlist above: the consent to write cleartext prompts to *this* disk +# belongs to the checkout, not the corpus, so a second clone must not inherit +# it. Absent means the installation-form default applies; `doctor` reports +# which. +/.wikitool-telemetry.json + # Coverage output from `pytest --cov` (see .gitea/workflows/ci.yml). Derived, # like reports/: recomputable from any commit, and `publish` runs `git add -A`, # so an unignored htmlcov/ would commit itself on the next content publish. diff --git a/CHANGES.md b/CHANGES.md index 1fb0aca..b7e0523 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -35,7 +35,7 @@ dev-checkout concern - readable here, never shipped as something to parse. --- -## 5.0.0-beta.8 - 2026-09-10 - CLAUDE.md-Importkette entdrifted, Modellwahl nach docs/ verschoben (schliesst #81) +## 5.0.0-beta.9 - 2026-09-10 - Telemetrie-Default nach Installationsform, Byte-Deckel und Session-Retention **Author:** Torben Nehmer @@ -61,6 +61,7 @@ dev-checkout concern - readable here, never shipped as something to parse. - version-parts.md dokumentiert den --migration-required-Ruecknahmepfad - wiki-status verweist auf session-setup.md (schliesst #84) - CLAUDE.md-Importkette entdrifted, Modellwahl nach docs/ verschoben (schliesst #81) +- Telemetrie-Default nach Installationsform, Byte-Deckel und Session-Retention @@ -1047,6 +1048,57 @@ nirgends geschah. `USER.md` und `SOUL.md` verlieren an derselben Stelle Rahmen- bzw. Herkunftsprosa, die `USER.md.template` bzw. ein Kommentar in `SOUL.md` selbst schon trägt. +**Telemetrie-Default nach Installationsform** (#55): `chemenu.telemetry.writer.enabled()` war +eine Zeile - immer an, `WIKI_TRACE=0` das einzige Opt-out. Richtig für dieses Repo, dessen +Traces das Messinstrument sind, mit dem der Stack sich selbst bewertet, aber die falsche +Voreinstellung für eine ausgelieferte Instanz: dort hat niemand Telemetrie bestellt, und +niemand liest `EVALS.md`, bevor die erste Datei geschrieben ist. Dazu kam eine zweite Lücke: +keine Mengenbegrenzung irgendeiner Art - `reports/telemetry//trace.jsonl` wächst, +solange die Instanz läuft, und nichts räumt je etwas weg. + +`tools/chemenu/telemetry/policy.py` (neu) löst jetzt beides an einer Stelle, gekeyt auf den +aufgelösten Root, damit `wikitool doctor`, der Writer und der MCP-Server-Start-Guard dieselbe +Antwort für denselben Checkout geben. Ein Git-Clone dieses Repos bleibt beim alten Verhalten +(an, `WIKI_TRACE=0` schaltet ab); eine per `dist export` ausgelieferte Instanz startet ab jetzt +mit Telemetrie **aus** - erkannt an der ohnehin vorhandenen, maschinengeschriebenen +`.wikitool-release.json` (Invariante 1). Wer sie dort anschalten will, legt eine +`.wikitool-telemetry.json` an (pro Checkout, gitignored, kein `.template` - wie +`.wikitool-remotes.json`); `WIKI_TRACE` überschreibt weiterhin beide Richtungen und schlägt die +Datei. + +Zwei unabhängige, fail-silent durchgesetzte Mengendeckel greifen in beiden Installationsformen: +ein Byte-Deckel pro Session-Trace (Default 5 MiB, ein `stat` vor jedem Append) und eine +Retention über die Anzahl der Session-Verzeichnisse (Default 250). Die Retention reserviert den +Platz der gerade entstehenden Session, statt sie mitzuzählen - sonst pendelt der Bestand +dauerhaft bei `keep+1` statt bei `keep`, weil jeder Lauf immer nur das räumt, was der *vorige* +Lauf über dem Limit gelassen hat. Am Byte-Limit schreibt ein weiterer Aufruf nichts mehr außer +einem einmaligen `telemetry.limit`-Event, per Exclusive-Create auf eine `.limit`-Sentinel-Datei +ausgelost - derselbe Ein-Schreiber-Trick wie beim `session.start`-Header, für den Fall, dass +mehrere Prozesse gleichzeitig auf denselben Trace schreiben. Eine Retention-Runde löscht +ausschließlich `trace.jsonl`/`.limit` der überzähligen Verzeichnisse und `rmdir`t nur, wenn +danach leer - nie `rmtree`, aus demselben Grund wie bei `upstream merge` (#30): +`reports/telemetry/` hält lokale, nicht rekonstruierbare Daten, die `eval score` liest. + +`wikitool doctor` bekommt einen neuen `telemetry`-Check (an/aus, warum, Menge gegen beide +Deckel, nie `FAIL`, wie `publish-remotes` und `environment`). Der MCP-Server-Start-Guard +(`check_trace_destination`) las `WIKI_TRACE` bisher selbst statt den Writer zu fragen - eine +zweite Kopie derselben Regel, die Invariante 8 verletzte, bevor diese Änderung sie schließt; er +fragt jetzt dieselbe Policy. `instructions/setup-instance.md` bekommt einen neuen +Entscheidungspunkt (Schritt 10). + +**MINOR, kein neuer Boundary-Crossing:** additiv in beide Richtungen - eine bestehende Instanz +kopiert die neue Maschinerie über sich und hört still auf zu schreiben, ohne Hand-Arbeit oder +Migration; die alte Version zurücklegen stellt den alten Default wieder her, weil sie die neue +Datei und die neuen Variablen schlicht ignoriert. Der Kandidat trägt seine `--breaking`-Zeile +bereits aus einem früheren Bump (Confidence-Entfernung); diese Änderung fügt keine neue hinzu. + +Geändert: `tools/chemenu/telemetry/policy.py` (neu), `tools/chemenu/telemetry/writer.py`, +`tools/chemenu/telemetry/schema.py`, `tools/chemenu/commands/doctor.py`, +`tools/chemenu/mcp/server.py`, `tools/chemenu/config.py`, `tools/chemenu/version.py`, +`.gitignore`, `tools/chemenu/tests/conftest.py` und die Telemetrie-/Doctor-/MCP-Server-Tests, +`.gitea/workflows/ci.yml`, `EVALS.md`, `INSTALL.md`, `INSTALL-MCP.md`, `reports/CONTRACT.md`, +`tools/CONTRACT.md`, `instructions/setup-instance.md`. Schließt #55. + --- ## 4.7.4 - 2026-09-04 - bootstrap.md nennt den session-id-WARN nach frischem Bootstrap explizit als erwartet diff --git a/EVALS.md b/EVALS.md index 6ee42d2..250fc64 100644 --- a/EVALS.md +++ b/EVALS.md @@ -190,6 +190,57 @@ What that verification turned up, and what it changes: `wikitool instructions sync` publishes is a project-scope skill source for Vibe, so this repository needs no adaptation to be worked on with it - only a trusted folder. +## Whether it runs at all + +The default depends on how this tree got here, not on a single hard-coded switch - +[tools/chemenu/telemetry/policy.py](tools/chemenu/telemetry/policy.py) is the one place that +resolves it, so `wikitool doctor`, the writer and the MCP server's start-up guard all answer the +same question the same way: + +| Installation form | Default | Marker | +|---|---|---| +| Git clone of this repo (dev checkout) | **on** (opt-out) | No `.wikitool-release.json` | +| `dist export` tarball (a distributed instance) | **off** (opt-in) | `.wikitool-release.json` present | + +The form is read off `.wikitool-release.json`, the same stamp `version check` and `dist upgrade` +already use to tell a distribution from the repo it came from - present means an operator never +asked for telemetry, absent means this is the dev checkout the stack ships from, where the traces +are its own measuring instrument (the rest of this file). A private instance +(`instructions/private-instance.md`) is a git clone of an *export*, so it carries the stamp and +defaults off too - it is a consuming instance, not a measuring stand. + +**Turning it on for a distributed instance** is a per-checkout `.wikitool-telemetry.json` at the +repo root, gitignored like `.wikitool-remotes.json` and for the same reason: the consent to write +cleartext prompts to *this* disk belongs to the checkout, not the corpus, so a second clone must +not inherit it silently. `instructions/setup-instance.md`'s Telemetry decision point asks for it +during setup; nothing writes it automatically. + +```json +{ "enabled": true, "max_session_bytes": 5242880, "keep_sessions": 250 } +``` + +All three keys are optional. `WIKI_TRACE` still overrides `enabled` in both directions and beats +the file, exactly as it always has. + +**Two independent quantity caps, both enforced fail-silent in `emit()`** - never in +`write_event()`, which the test suite calls directly to exercise the format without the policy +wrapped around it: + +- **A byte cap per session trace** (default 5 MiB, `max_session_bytes` / `WIKI_TRACE_MAX_SESSION_BYTES`), + checked with one `stat` before every append. Once a trace is at or over the cap, further calls + in that session write nothing except a single `telemetry.limit` event - elected by the same + single-writer trick `session.start` uses (an exclusive-create on a `.limit` marker file), so a + trace that was cut off is distinguishable from one whose writer simply crashed. +- **Retention by session count** (default 250, `keep_sessions` / `WIKI_TRACE_KEEP_SESSIONS`), + applied once, right before a brand-new session directory is created - never per event, and + never against the session doing the creating. It deletes exactly `trace.jsonl` and `.limit` + from the oldest directories beyond the cap and only `rmdir`s one once it is empty; nothing + under `reports/` is ever removed in bulk. + +The default of 250 is chosen above what this repo's own checkout has accumulated as of +2026-09-10 (231 session directories, well under 400 KiB total) - the cap starts biting on future +growth, not on the existing history. + ## What never reaches a trace Prompts and assistant replies **are** recorded in cleartext, locally. A failure taxonomy @@ -214,10 +265,12 @@ may only be enabled against a collector you run yourself. | Variable | Effect | |---|---| -| `WIKI_TRACE=0` | Record nothing | +| `WIKI_TRACE` | `0`/`1` overrides on/off in either direction, beating both the installation-form default and `.wikitool-telemetry.json` - see § Whether it runs at all | | `WIKI_TRACE_DIR` | Write traces somewhere other than `reports/telemetry/` | | `WIKI_TRACE_CONTENT=0` | Lengths and digests instead of text | | `WIKI_TRACE_MAX_CONTENT` | Per-attribute cap in characters | +| `WIKI_TRACE_MAX_SESSION_BYTES` | Per-session trace byte cap (default 5 MiB) - overrides `.wikitool-telemetry.json`'s `max_session_bytes` | +| `WIKI_TRACE_KEEP_SESSIONS` | How many session directories retention keeps (default 250) - overrides `.wikitool-telemetry.json`'s `keep_sessions` | | `WIKITOOL_SESSION_ID` | The join key, and the directory a trace lands in | ## Evaluation levels diff --git a/INSTALL-MCP.md b/INSTALL-MCP.md index 7aa74a8..02168e7 100644 --- a/INSTALL-MCP.md +++ b/INSTALL-MCP.md @@ -43,9 +43,13 @@ WIKI_TRACE=0 tools/.venv/bin/python -m chemenu.mcp Der Prozess spricht MCP über stdin/stdout und gibt für sich genommen nichts aus — das ist richtig so. Gestartet wird er normalerweise nicht von Hand, sondern vom Client (Schritt 3). -**`WIKI_TRACE=0` ist nicht optional.** Telemetrie ist per Default an und schreibt nach -`reports/telemetry/` im Repo. Der Server **verweigert den Start**, solange das so ist, statt -still umzuleiten: +**Das `WIKI_TRACE=0` oben ist in einer ausgelieferten Instanz meist redundant, aber trotzdem +richtig.** Telemetrie-Default hängt vom Installationsweg ab +([INSTALL.md](INSTALL.md) § Konfiguration): in einer per `dist export` ausgelieferten Instanz +steht er auf **aus**, in einem Git-Clone dieses Repos (Testbett/Demo) auf **an**. Der Server +prüft nicht `WIKI_TRACE` direkt, sondern denselben `chemenu.telemetry.policy`, den auch +`wikitool doctor` und der Writer befragen - läuft Telemetrie danach, **verweigert der Server den +Start**, statt still umzuleiten: ``` ERROR Telemetry is on and would write into the served checkout (...). Set WIKI_TRACE=0, @@ -53,7 +57,10 @@ or point WIKI_TRACE_DIR outside the corpus. ``` Beide Auswege sind gleichwertig: `WIKI_TRACE=0` schaltet ab, `WIKI_TRACE_DIR=/var/log/chemenu` -lenkt um. Der Grund steht in Schritt 6 — der Sync darf `reports/` wegräumen. +lenkt um. Der Grund steht in Schritt 6 — der Sync darf `reports/` wegräumen. Das explizite +`WIKI_TRACE=0` in den Befehlen dieses Dokuments bleibt der sichere Default: es ist korrekt, egal +welchen Weg die bediente Instanz genommen hat, und macht die Prüfung oben gegenstandslos, statt +sich auf den Installationsweg zu verlassen. ## Schritt 3: Einen Client einbinden diff --git a/INSTALL.md b/INSTALL.md index b0dd588..ad8b42f 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -283,11 +283,32 @@ Ausnahmen (`kb/CONVENTIONS.md`, `kb/*/COLLECTION.md`, `.wikitool-kb.json`) in | `WIKITOOL_UPDATE_URL` | Release-Feed, den `version check` abfragt | Wert aus `.wikitool-release.json`, sonst der Feed der Ursprungs-Instanz | | `WIKITOOL_UPDATE_TOKEN` | Gitea-Token für den Release-Feed | keiner - gegen `torben/chemenu` nicht nötig, nur für einen privaten Fork (siehe unten) | | `CHEMENU_ROOT` | Auf welchen Korpus das Paket zeigt - für einen Aufrufer, der nicht im Checkout selbst liegt | der Checkout, in dem das Paket liegt (`tools/wikitool` verhält sich ohne die Variable unverändert) | -| `WIKI_TRACE` / `WIKI_TRACE_DIR` | Telemetrie abschalten bzw. aus dem Arbeitsbaum heraus umlenken | an, nach `reports/telemetry/` - der MCP-Server verweigert damit den Start, siehe [INSTALL-MCP.md](INSTALL-MCP.md) | +| `WIKI_TRACE` / `WIKI_TRACE_DIR` | Telemetrie abschalten bzw. aus dem Arbeitsbaum heraus umlenken | Hängt vom Installationsweg ab - siehe unten | +| `WIKI_TRACE_MAX_SESSION_BYTES` / `WIKI_TRACE_KEEP_SESSIONS` | Byte-Deckel je Session-Trace bzw. wie viele Session-Verzeichnisse die Retention behält | 5 MiB je Session, 250 Verzeichnisse | **Gegen das Ursprungs-Repo braucht es kein Token.** `torben/chemenu` ist öffentlich lesbar; `version check` und der Download in Weg A funktionieren ohne Konfiguration. +**Telemetrie-Default hängt vom gewählten Weg ab, nicht von einem festen Schalter.** Weg A und +Weg B erzeugen eine `.wikitool-release.json` (Weg A trägt sie schon im Release, Weg B schreibt +sie beim Export) - daran erkennt `chemenu.telemetry.policy`, dass niemand Telemetrie bestellt +hat, und der Default steht auf **aus**. Weg C (dieses Repo geklont) trägt keine solche Datei - +hier sind die Traces das Messinstrument, mit dem der Stack sich selbst bewertet, und der +Default steht auf **an**. Weg D erbt den Default von der Distribution, aus der die private +Instanz entstand, also ebenfalls **aus**. + +Wer den Default umdrehen will, legt `.wikitool-telemetry.json` im Repo-Root an (pro Checkout, +gitignored, kein `.template` - genau wie `.wikitool-remotes.json`): + +```json +{ "enabled": true, "max_session_bytes": 5242880, "keep_sessions": 250 } +``` + +Alle drei Schlüssel sind optional. `WIKI_TRACE` überschreibt `enabled` weiterhin in beide +Richtungen und schlägt diese Datei. `wikitool doctor` meldet den aktuellen Zustand (an/aus, +warum, und die Menge gegen beide Deckel); mehr dazu in [EVALS.md](EVALS.md) § "Whether it +runs at all". + **Für einen privaten Fork schon.** Wer den Stack in ein eigenes, nicht öffentliches Repo legt und `WIKITOOL_UPDATE_URL` auf dessen Feed zeigen lässt, stößt auf eine Eigenheit, die man kennen sollte: Gitea antwortet anonymen Aufrufern für ein unsichtbares Repo mit demselben diff --git a/VERSION b/VERSION index 1a9fa50..0f5608e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.0-beta.8 +5.0.0-beta.9 diff --git a/instructions/setup-instance.md b/instructions/setup-instance.md index b79ac08..08c7e41 100644 --- a/instructions/setup-instance.md +++ b/instructions/setup-instance.md @@ -147,7 +147,7 @@ bereit für den ersten `Ingest`. identifiers](../kb/CONTRACT.md#language-and-identifiers)). Titel, Wikilink-Ziele, Cite-IDs, Enum-Werte, Tags, Befehle und Pfade folgen keiner KB-Sprache. - `tools/wikitool doctor` prüft das Ergebnis in Schritt 12 (`conventions`): eine fehlende + `tools/wikitool doctor` prüft das Ergebnis in Schritt 13 (`conventions`): eine fehlende Datei ist ein `FAIL`, eine mit Sentinel oder ohne vollständigen `sections:`-Block ebenso. `docs verify` prüft zusätzlich `profile:` und `required_by_stack:` auf jedem `COLLECTION.md`. @@ -184,7 +184,7 @@ bereit für den ersten `Ingest`. Invariante 3. Sie ändern keine Regel aus [AGENTS.md](../AGENTS.md), und eine Nutzeraussage wandert daraus nie ohne den normalen Quelle/Provenance-Prozess nach `kb/`. - `tools/wikitool doctor` prüft das Ergebnis in Schritt 12 (`personalization`): eine fehlende + `tools/wikitool doctor` prüft das Ergebnis in Schritt 13 (`personalization`): eine fehlende Datei ist ein `FAIL`, eine, die noch den Sentinel trägt, ebenso - ein umbenanntes Template ist kein ausgefülltes. @@ -215,24 +215,49 @@ bereit für den ersten `Ingest`. Schreiben die Sentinel-Zeile (`wikitool:template-unfilled`) entfernen; das `.template` bleibt liegen. - Wird der Schritt übersprungen, läuft alles weiter: `doctor` meldet in Schritt 12 + Wird der Schritt übersprungen, läuft alles weiter: `doctor` meldet in Schritt 13 `environment: absent (optional)`, kein `FAIL`. Die Datei ist gitignored und geht in keinen Commit ein - sie beschreibt diesen Checkout, nicht das Repo. -10. **Session-Budget scopen** (Details: [session-setup.md](session-setup.md)): +10. **Entscheidungspunkt - Telemetrie.** Der Default hängt am Installationsweg, nicht an + diesem Schritt: eine per `dist export` ausgelieferte Instanz - jede, die hier ankommt, ohne + Weg C (direkter Klon des Ursprungs-Repos) genommen zu haben - trägt eine + `.wikitool-release.json` und startet mit Telemetrie **aus**; niemand hat sie bestellt, und + `EVALS.md` liest ohnehin niemand, bevor die erste Datei geschrieben ist. Dieser Schritt + fragt nur, ob der Betreiber das umdrehen will. + + Den Nutzer einmal fragen: Telemetrie an? Falls ja, `.wikitool-telemetry.json` im + Repo-Root anlegen (pro Checkout, gitignored, kein `.template` - wie + `.wikitool-remotes.json`): + + ```json + { "enabled": true } + ``` + + `max_session_bytes` (Default 5 MiB) und `keep_sessions` (Default 250) sind optional in + derselben Datei; die meisten Instanzen brauchen sie nicht anzufassen. Falls nein, nichts + tun - der Default steht bereits auf aus, und keine Datei entsteht. `WIKI_TRACE` + überschreibt beide Richtungen weiterhin, falls eine einzelne Session abweichen soll. + + `tools/wikitool doctor` meldet das Ergebnis in Schritt 13 (`telemetry`): an/aus, warum + (Installationsform, diese Datei, oder `WIKI_TRACE`), und die aktuelle Menge gegen beide + Deckel - nie ein `FAIL`, in beide Richtungen ist das ein gültiger Zustand. Mehr dazu: + [EVALS.md](../EVALS.md) § "Whether it runs at all". + +11. **Session-Budget scopen** (Details: [session-setup.md](session-setup.md)): ```bash export WIKITOOL_SESSION_ID="wiki-$(date +%s)" ``` -11. **Generierte Indizes erzeugen** - `dist export` liefert sie bewusst nicht mit: +12. **Generierte Indizes erzeugen** - `dist export` liefert sie bewusst nicht mit: ```bash tools/wikitool index rebuild tools/wikitool sources rebuild-index ``` -12. **Verifizieren**, in dieser Reihenfolge: +13. **Verifizieren**, in dieser Reihenfolge: ```bash tools/wikitool doctor @@ -245,7 +270,7 @@ bereit für den ersten `Ingest`. keine `WIKITOOL_SESSION_ID`) ist kein Blocker. Ein `FAIL` benennt sein eigenes Fix-Kommando; das ausführen und `doctor` erneut aufrufen. -13. **Ersten Commit anstoßen:** +14. **Ersten Commit anstoßen:** ```bash tools/wikitool publish --message "chore: initial instance setup" @@ -257,7 +282,7 @@ bereit für den ersten `Ingest`. Dateiliste und die exakte `--confirm `-Zeile, die nach seiner Freigabe veröffentlicht. Details zum Gate: [gates.md](gates.md). -14. **Agent-Session neu starten.** Harnesses lesen die Skill-Verzeichnisse beim Start; erst +15. **Agent-Session neu starten.** Harnesses lesen die Skill-Verzeichnisse beim Start; erst danach sind `wiki-ingest`, `wiki-query`, `wiki-manage`, `wiki-lint` und `wiki-status` verfügbar. diff --git a/reports/CONTRACT.md b/reports/CONTRACT.md index 5fbbc1b..f57e61b 100644 --- a/reports/CONTRACT.md +++ b/reports/CONTRACT.md @@ -33,11 +33,21 @@ prompts and assistant replies in cleartext and belongs to the machine it ran on, repository. What a trace concludes - a scored eval, a failure taxonomy - is carried out the same way a lint report's judgment is: into `kb/`, `work/`, or `kb/log.md`. -`WIKI_TRACE_DIR` redirects the tree; `WIKI_TRACE=0` turns recording off; `WIKI_TRACE_CONTENT=0` -keeps lengths and digests instead of text. +`WIKI_TRACE_DIR` redirects the tree; `WIKI_TRACE` turns recording on or off, overriding both the +installation-form default (on for a dev checkout, off for a distributed instance - +`chemenu.telemetry.policy`) and a per-checkout `.wikitool-telemetry.json`; `WIKI_TRACE_CONTENT=0` +keeps lengths and digests instead of text. See [../EVALS.md](../EVALS.md) § "Whether it runs at +all" for the full precedence and both quantity caps below. ## Retention -None. Old reports are local scratch; delete them freely. There is nothing to retire with -`wikitool rm`, because no report is ever a wiki page - `lint-report` is a contract-only -type-spec with no `base_dir:` and cannot be instantiated under `kb/`. +**Lint reports: none.** Old ones are local scratch; delete them freely. There is nothing to +retire with `wikitool rm`, because no report is ever a wiki page - `lint-report` is a +contract-only type-spec with no `base_dir:` and cannot be instantiated under `kb/`. + +**Traces: two enforced caps, applied by the writer itself, never by a separate cleanup pass.** +A byte cap per session trace (default 5 MiB, `WIKI_TRACE_MAX_SESSION_BYTES`) and a retention +limit on the number of session directories under `reports/telemetry/` (default 250, +`WIKI_TRACE_KEEP_SESSIONS`), both fail-silent like the writer itself. Neither ever removes a +session directory in bulk - only the `trace.jsonl`/`.limit` files a cap's own rule names, and +only once the directory is otherwise empty. diff --git a/tools/CONTRACT.md b/tools/CONTRACT.md index b2150c1..65fd9eb 100644 --- a/tools/CONTRACT.md +++ b/tools/CONTRACT.md @@ -103,7 +103,7 @@ never ran this step), `VERSION`, `USER.md.template`/`SOUL.md.template` plus `kb/ | `migrate baseline [--force]` | Declare `kb_version` once, for an instance predating `.wikitool-kb.json`. Refuses to overwrite an existing declaration without `--force`: advancing after a migration is `done`, which checks the chain, and this command must not become the quiet way around it | | `upstream merge [--remote upstream] [--branch main] [--no-fetch]` | Take a stack update into a private instance's branch, machinery only - the code procedure behind `instructions/private-instance.md` § "Taking a stack update". Refuses on a dirty working tree, a merge already in progress, or a remote that does not resolve; WARNs (does not block) when `.wikitool-remotes.json` is absent, pointing at the setup step that arms it. Fetches `/` (unless `--no-fetch`) and reports "already up to date" if nothing new exists. Otherwise opens `git merge --no-commit --no-ff /` - and stops, untouched, if git refused to open a merge at all (unrelated histories), since without a `MERGE_HEAD` every stack path would read as "the upstream deleted it". Then forces every content stage (`kb/`, `raw/`, `work/`, `reports/`) back to the local side by removing **only the paths tracked in either tree** and checking `HEAD`'s back out - never the stage directory wholesale, because `reports/` is gitignored apart from its contract and holds local, non-recomputable data (telemetry traces `eval score` reads, saved eval and lint reports) that no merge has business deleting. Then restores from the upstream side exactly the paths `chemenu.ownership.is_stack_owned` recognises as machinery (`/CONTRACT.md`, and anything ending `.template` under a content stage) - including a deletion, if the upstream removed one. A real conflict left in `tools/`, `types/` or `instructions/` after that leaves the merge open, uncommitted, and exits 1 rather than guessing. Commits with `git commit --no-edit`, then re-checks the resulting range with the same logic as `upstream verify`; a finding there is a loud, uncommitted-nothing-rolled-back error, because the merge commit already exists and needs a human's eyes, not an automatic repair. Never pushes. Not idempotent - see the tool error contract below | | `upstream verify --since [--until HEAD]` | Compare two revisions: did anything under a content stage (`kb/`, `raw/`, `work/`, `reports/`) change except through a stack-owned path? Shares its check with `upstream merge`'s own postcheck, so a hand-resolved merge conflict, or a `dist upgrade`, can be verified the same way. Exit 1 with the offending paths if anything leaked; otherwise reports which stack-owned paths legitimately moved. Read-only and exempt from the Iteration Budget Gate, like `migrate verify` | -| `doctor [--json]` | Check that this instance is correctly configured: dependencies (Python, ripgrep), author resolution, stack version, git identity/branch/remote, published skills, kb/raw/reports/work/instructions structure, personalization (`USER.md`/`SOUL.md` present **and** filled - a file still carrying the template's sentinel is a `FAIL`, since a renamed template is not a filled one), the KB conventions (`kb/CONVENTIONS.md` present, unsentinelled, and naming all three tool-owned section headings - a `FAIL` on any of the three, because `xref`/`cite` write out of it), the environment note (`ENVIRONMENT.md` - optional, so absent is `OK`; a still-templated one is a `WARN`), generated files, and `WIKITOOL_SESSION_ID`. Read-only, exit 1 only on a `FAIL` (a missing remote, session id, or `VERSION` is a `WARN`, not a fault). Exempt from the Iteration Budget Gate | +| `doctor [--json]` | Check that this instance is correctly configured: dependencies (Python, ripgrep), author resolution, stack version, git identity/branch/remote, published skills, kb/raw/reports/work/instructions structure, personalization (`USER.md`/`SOUL.md` present **and** filled - a file still carrying the template's sentinel is a `FAIL`, since a renamed template is not a filled one), the KB conventions (`kb/CONVENTIONS.md` present, unsentinelled, and naming all three tool-owned section headings - a `FAIL` on any of the three, because `xref`/`cite` write out of it), the environment note (`ENVIRONMENT.md` - optional, so absent is `OK`; a still-templated one is a `WARN`), generated files, `WIKITOOL_SESSION_ID`, and telemetry state (on/off, why - installation-form default, `.wikitool-telemetry.json`, or `WIKI_TRACE` - and the current session count/byte total against both caps; never `FAIL`, see [EVALS.md](../EVALS.md)). Read-only, exit 1 only on a `FAIL` (a missing remote, session id, or `VERSION` is a `WARN`, not a fault). Exempt from the Iteration Budget Gate | ## Design notes @@ -221,7 +221,7 @@ is atomic, and whether a retry is safe. | `doctor` | At least one check reported `FAIL` (a `WARN`, e.g. no remote or no `WIKITOOL_SESSION_ID`, does not exit 1) | Read-only | Each finding names its own fix command; re-run after applying it | | `budget status` / `budget reset` | `reset` without `--yes`; `status` never fails | Read/rewrite of one JSON file | `status` is safe to retry. For `reset`: get the user's approval, then re-run with `--yes` | | `eval sessions` | Never fails; an empty list is a valid answer | Read-only | - | -| `eval score` | No trace exists for the named session | Read-only, apart from the files `--save`/`--markdown` write | Run `eval sessions` to see which ids exist. A session records nothing when `WIKI_TRACE=0`, so an absent trace is not necessarily a fault. Safe to retry | +| `eval score` | No trace exists for the named session | Read-only, apart from the files `--save`/`--markdown` write | Run `eval sessions` to see which ids exist. A session records nothing when telemetry is off - `WIKI_TRACE=0`, or a distributed instance with no opt-in (`wikitool doctor` says which) - so an absent trace is not necessarily a fault. Safe to retry | | *(any command)* - Iteration Budget Gate / Loop-Breaker | Session call limit exceeded, or the last 3 calls were identical | N/A - pre-dispatch check, the command never ran | **Not** safe to retry as-is; retrying is the failure mode being prevented. Stop and escalate | ## Maintenance schedule diff --git a/tools/chemenu/commands/doctor.py b/tools/chemenu/commands/doctor.py index e63cd1c..3cf8a99 100644 --- a/tools/chemenu/commands/doctor.py +++ b/tools/chemenu/commands/doctor.py @@ -345,6 +345,37 @@ def check_generated_files() -> Check: return Check("generated-files", "OK", "kb/index.md, kb/log.md, kb/provenance.md present") +def check_telemetry() -> Check: + """Whether tracing is on for this checkout, why, and how full its two caps + are. + + Never `FAIL`s, the same line `check_publish_remotes` and + `check_environment` draw: both an enabled and a disabled tree are + legitimate states, and a `FAIL` would make one of them mandatory by the + back door. The tree walk below is fine here - `doctor` is not a hot path, + unlike the `stat` `emit()` does on every append. + """ + from chemenu.telemetry import policy as telemetry_policy + from chemenu.telemetry import reader + from chemenu.telemetry.writer import TRACE_FILE, trace_root + + pol = telemetry_policy.resolve(config.ROOT) + root = trace_root() + session_count = len(reader.sessions(root)) + total_bytes = ( + sum(f.stat().st_size for f in root.glob(f"*/{TRACE_FILE}") if f.is_file()) + if root.exists() + else 0 + ) + state = "on" if pol.enabled else "off" + return Check( + "telemetry", "OK", + f"{state} ({pol.reason}); {session_count}/{pol.keep_sessions} session(s), " + f"{total_bytes:,} byte(s) under {rel_path(root)} " + f"(cap {pol.max_session_bytes:,} byte(s)/session)", + ) + + def check_session_id() -> Check: import os @@ -449,6 +480,7 @@ def run_doctor() -> list[Check]: check_publish_remotes(), check_generated_files(), check_session_id(), + check_telemetry(), ] return checks @@ -458,7 +490,8 @@ def doctor_command( ): """Check that this instance is correctly configured: dependencies, author, git identity/remote, published skills, structure, personalization, KB - conventions, generated files, and session scoping. Read-only. Exits 1 only + conventions, generated files, session scoping, and telemetry state. + Read-only. Exits 1 only if a check FAILs.""" checks = run_doctor() diff --git a/tools/chemenu/config.py b/tools/chemenu/config.py index fb48639..0147d09 100644 --- a/tools/chemenu/config.py +++ b/tools/chemenu/config.py @@ -202,6 +202,24 @@ ENVIRONMENT_TEMPLATE = f"{ENVIRONMENT_FILE}.template" # `dist export` already computes (AGENTS.md invariant 8). See NOTICE. LICENSE_FILES = ("LICENSE", "LICENSE-CONTENT", "NOTICE") +# Written by `dist export` into every distribution; presence means "this tree +# is an exported instance", absence means "this is the dev checkout the stack +# ships from". Defined here rather than only in `version.py`, because +# `chemenu.telemetry` (`telemetry/policy.py`) needs the same marker to answer +# opt-in vs. opt-out and must stay importable without the venv - a hook +# handler imports it on every tool call. `version.py` re-exports this name +# rather than defining its own, per AGENTS.md invariant 8. +RELEASE_STAMP_FILENAME = ".wikitool-release.json" + +# Per-checkout telemetry opt-in/opt-out plus its two quantity caps (see +# `telemetry/policy.py`). Same shape as `PUBLISH_REMOTES_FILENAME` below: it +# answers a question about *this* checkout, so it is per-checkout and +# gitignored, ships no `.template`, and its absence is a legitimate state - +# the installation-form default (keyed off `RELEASE_STAMP_FILENAME`) applies. +# `instructions/setup-instance.md`'s Telemetry decision point writes it from +# the operator's answer. +TELEMETRY_FILENAME = ".wikitool-telemetry.json" + # Which push targets `publish` may write to, for a checkout that says so. The # danger this addresses is one checkout's content reaching another checkout's # remote - a private instance pushing its own `kb/` to a public upstream, where diff --git a/tools/chemenu/mcp/server.py b/tools/chemenu/mcp/server.py index ba642d4..904245b 100644 --- a/tools/chemenu/mcp/server.py +++ b/tools/chemenu/mcp/server.py @@ -43,6 +43,7 @@ from mcp.server.mcpserver.exceptions import ToolError from chemenu import config from chemenu.api import Corpus from chemenu.errors import ChemenuError +from chemenu.telemetry import policy SERVER_NAME = "chemenu" @@ -62,21 +63,24 @@ class TraceWouldWriteIntoCorpus(RuntimeError): def check_trace_destination(root: Path) -> None: """Refuse to start if a trace would be written into the corpus. - Telemetry defaults to *on* and writes under `reports/telemetry/` in the - repo. Today nothing on this path emits - the writer is wired into - `cli.main()` and the two gates, none of which run here - so this is a guard - against the future rather than a fix for the present. It is worth having - anyway: the sync that keeps this checkout current is `git reset --hard`, so - a trace written into the tree is both a per-request write into a directory - something else is entitled to wipe, and a silent way for the server to - dirty the tree its own cache keys on. + Telemetry's own default now depends on the installation form + (`chemenu.telemetry.policy`) - off for a distributed instance, on for a + dev checkout - so this guard resolves the same policy rather than reading + `WIKI_TRACE` a second time, which is how it used to drift from the writer + (AGENTS.md invariant 8). Today nothing on this path emits - the writer is + wired into `cli.main()` and the two gates, none of which run here - so + this is a guard against the future rather than a fix for the present. It + is worth having anyway: the sync that keeps this checkout current is + `git reset --hard`, so a trace written into the tree is both a per-request + write into a directory something else is entitled to wipe, and a silent + way for the server to dirty the tree its own cache keys on. Turn tracing off (`WIKI_TRACE=0`) or point it somewhere else (`WIKI_TRACE_DIR`). Refusing rather than correcting it: a server that quietly relocates the operator's telemetry is a surprise waiting in a log nobody reads. """ - if os.environ.get("WIKI_TRACE", "1") == "0": + if not policy.resolve(root).enabled: return destination = os.environ.get("WIKI_TRACE_DIR") if destination is None: diff --git a/tools/chemenu/telemetry/policy.py b/tools/chemenu/telemetry/policy.py new file mode 100644 index 0000000..7212a52 --- /dev/null +++ b/tools/chemenu/telemetry/policy.py @@ -0,0 +1,155 @@ +"""Telemetry on/off and its two quantity caps, resolved once per root. + +One place to answer three questions, because `writer.py`, `wikitool doctor` +and the MCP server's start-up guard must agree on the same answer for the +same checkout - a second copy of this logic is exactly how the MCP server +used to drift from the writer (it read `WIKI_TRACE` itself; fixed alongside +this module, see AGENTS.md invariant 8). + +**Precedence for `enabled`:** `WIKI_TRACE` (either direction) beats +`.wikitool-telemetry.json`, which beats the installation-form default. The +form is read off `config.RELEASE_STAMP_FILENAME`: present means an exported/ +distributed tree (default *off* - an operator never asked for telemetry), +absent means the dev checkout this package ships from (default *on* - the +traces are this stack's own measuring instrument, see EVALS.md). + +**Caching.** Only the filesystem-derived half - the release stamp and the +config file - is cached, keyed on the resolved root. `WIKI_TRACE` and the two +`WIKI_TRACE_MAX_SESSION_BYTES`/`WIKI_TRACE_KEEP_SESSIONS` overrides are read +fresh on every call: they are cheap (no I/O) and a test that flips one +mid-run must see the new value immediately, not a cached one. A test that +rewrites the release stamp or the config file at an already-resolved root +calls `reset_cache()` itself, the same discipline `conventions.reset_cache()` +already follows. + +Stdlib only, like the rest of `chemenu.telemetry` - a hook handler imports +this package on every tool call and must not need the venv. +""" +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path + +from chemenu import config + +DEFAULT_MAX_SESSION_BYTES = 5 * 1024 * 1024 +DEFAULT_KEEP_SESSIONS = 250 + +ENV_ENABLED = "WIKI_TRACE" +ENV_MAX_SESSION_BYTES = "WIKI_TRACE_MAX_SESSION_BYTES" +ENV_KEEP_SESSIONS = "WIKI_TRACE_KEEP_SESSIONS" + + +@dataclass(frozen=True) +class Policy: + enabled: bool + reason: str # human-readable - what decided `enabled`, for `doctor` + max_session_bytes: int + keep_sessions: int + + +@dataclass(frozen=True) +class _RootDefaults: + form_default: bool + form_reason: str + file_enabled: bool | None + file_reason: str | None + file_max_bytes: int | None + file_keep_sessions: int | None + + +_cache: dict[Path, _RootDefaults] = {} + + +def reset_cache() -> None: + _cache.clear() + + +def _positive_int(value: object) -> int | None: + try: + parsed = int(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return None + return parsed if parsed > 0 else None + + +def _read_config(root: Path) -> dict: + try: + raw = (root / config.TELEMETRY_FILENAME).read_text(encoding="utf-8") + except OSError: + return {} + try: + data = json.loads(raw) + except json.JSONDecodeError: + return {} + return data if isinstance(data, dict) else {} + + +def _root_defaults(root: Path) -> _RootDefaults: + cached = _cache.get(root) + if cached is not None: + return cached + + distribution = (root / config.RELEASE_STAMP_FILENAME).is_file() + form_default = not distribution + form_reason = ( + f"no {config.RELEASE_STAMP_FILENAME} - dev checkout, default on" + if form_default + else f"{config.RELEASE_STAMP_FILENAME} present - distributed instance, default off" + ) + + data = _read_config(root) + file_enabled = bool(data["enabled"]) if "enabled" in data else None + file_reason = ( + f"{config.TELEMETRY_FILENAME} sets enabled={file_enabled}" + if file_enabled is not None + else None + ) + + result = _RootDefaults( + form_default=form_default, + form_reason=form_reason, + file_enabled=file_enabled, + file_reason=file_reason, + file_max_bytes=_positive_int(data.get("max_session_bytes")), + file_keep_sessions=_positive_int(data.get("keep_sessions")), + ) + _cache[root] = result + return result + + +def resolve(root: "Path | str | None" = None) -> Policy: + """The effective policy for `root` (default: `config.ROOT`).""" + resolved_root = Path(root).resolve() if root is not None else Path(config.ROOT).resolve() + defaults = _root_defaults(resolved_root) + + env_enabled = os.environ.get(ENV_ENABLED) + if env_enabled is not None: + enabled = env_enabled != "0" + reason = f"{ENV_ENABLED}={env_enabled!r} overrides" + elif defaults.file_enabled is not None: + enabled = defaults.file_enabled + reason = defaults.file_reason # type: ignore[assignment] + else: + enabled = defaults.form_default + reason = defaults.form_reason + + max_bytes = ( + _positive_int(os.environ.get(ENV_MAX_SESSION_BYTES)) + or defaults.file_max_bytes + or DEFAULT_MAX_SESSION_BYTES + ) + keep_sessions = ( + _positive_int(os.environ.get(ENV_KEEP_SESSIONS)) + or defaults.file_keep_sessions + or DEFAULT_KEEP_SESSIONS + ) + + return Policy( + enabled=enabled, + reason=reason, + max_session_bytes=max_bytes, + keep_sessions=keep_sessions, + ) diff --git a/tools/chemenu/telemetry/schema.py b/tools/chemenu/telemetry/schema.py index b47e8f9..f9da4d0 100644 --- a/tools/chemenu/telemetry/schema.py +++ b/tools/chemenu/telemetry/schema.py @@ -57,6 +57,7 @@ OPTIONAL_EVENTS = frozenset( "publish.commit", "budget.state", "gate.cleared", + "telemetry.limit", } ) diff --git a/tools/chemenu/telemetry/writer.py b/tools/chemenu/telemetry/writer.py index 91001a5..cf11a17 100644 --- a/tools/chemenu/telemetry/writer.py +++ b/tools/chemenu/telemetry/writer.py @@ -1,17 +1,26 @@ """Appends trace events to `reports/telemetry//trace.jsonl`. -Two rules govern this module. +Three rules govern this module. **Telemetry never breaks the tool.** `emit()` swallows everything: a full disk, a permission error or a bug in a scrubber pattern must not turn a working `wikitool` command - or a hook wrapped around someone's tool call - into a -failure. Tests call `write_event()` instead, which raises. +failure. Tests call `write_event()` instead, which raises - and, unlike +`emit()`, bypasses policy entirely: no `enabled()` check, no byte cap, no +retention. A test wanting those exercises `emit()`. **Append, do not rewrite.** The budget state is a whole-file document and is written with the temp-file + `os.replace` dance. A trace is append-only, so the equivalent guarantee is `O_APPEND` plus an exclusive lock: several processes write to one trace at once (the CLI in one, a hook handler per tool call in another), and a line must never land inside another line. + +**Two quantity caps, both enforced in `emit()`, both fail-silent.** A byte cap +per session trace (`policy.max_session_bytes()`), checked with one `stat` +before every append; and a retention pass over `trace_root()`'s +session directories, run only when a brand-new one is about to be created - +never per event. See `chemenu.telemetry.policy` for where the numbers come +from. """ from __future__ import annotations @@ -22,18 +31,26 @@ from pathlib import Path from chemenu import config from chemenu.session import session_id as current_session_id from chemenu.session import session_slug -from chemenu.telemetry import schema, scrub +from chemenu.telemetry import policy, schema, scrub TRACE_ROOT = config.REPORTS_DIR / "telemetry" TRACE_FILE = "trace.jsonl" +# The single-writer sentinel that elects which process records the +# `telemetry.limit` marker for a session that has hit its byte cap - see +# `_mark_limit_once`. Also what `_enforce_retention` deletes alongside +# `trace.jsonl`: these two names are everything this policy owns per session +# directory. +LIMIT_MARKER = ".limit" +LIMIT_EVENT = "telemetry.limit" + # Per-process counter. Not comparable across processes - see schema.py on how # to order a trace. _seq = 0 def enabled() -> bool: - return os.environ.get("WIKI_TRACE", "1") != "0" + return policy.resolve(config.ROOT).enabled def trace_root() -> Path: @@ -167,6 +184,64 @@ def _locked_write(handle, line: str) -> None: fcntl.flock(handle, fcntl.LOCK_UN) +def _enforce_retention(root: Path, keep: int, exclude: str) -> None: + """Delete the oldest session directories' `trace.jsonl`/`.limit` so that, + once the session about to be created lands, at most `keep` remain. + + Runs once, right before a brand-new session directory would be created - + never per event, so an active session is never re-scanned for its own + writes. It reserves that new session's own slot up front (keeping + `keep - 1` of what already exists) rather than trimming to `keep` and then + letting the new one land as `keep + 1`: the latter never actually + converges back to `keep` under a steady trickle of new sessions, one at a + time, each pass only ever removing what the *previous* pass left over the + limit. Sorted by `mtime`, most recent first, the same order + `reader.sessions()` reads a trace root back in, so "oldest" here and + "most recent" there agree. + + `exclude` is the session about to be created - it does not exist on disk + yet in the common case, but is named explicitly anyway so a session whose + directory a concurrent process just created is never the one this pass + removes, regardless of its mtime. + + Deletes only the two files this policy owns per session; the directory + itself is `rmdir`-ed only once empty, never `rmtree`-d. A foreign file + left by something else in a session's directory keeps that directory, and + everything else in it, standing - `reports/` holds local, non-recomputable + data no retention pass has business deleting wholesale. + """ + if not root.is_dir(): + return + candidates = [p for p in root.iterdir() if p.is_dir() and p.name != exclude] + candidates.sort(key=lambda p: p.stat().st_mtime, reverse=True) + for stale in candidates[max(keep - 1, 0):]: + for name in (TRACE_FILE, LIMIT_MARKER): + try: + (stale / name).unlink() + except FileNotFoundError: + pass + try: + stale.rmdir() + except OSError: + pass # not empty - something else lives here, leave it standing + + +def _mark_limit_once(session_dir: Path, source: str, session: str, limit: int) -> None: + """Record `telemetry.limit` exactly once per session, however many further + calls hit the cap afterwards. + + The single-writer trick is the same one `_seed_session_header` uses: + several processes can hit the cap on the same trace at once, and an + `exists()` check would let more than one of them win. + """ + try: + handle = open(session_dir / LIMIT_MARKER, "x", encoding="utf-8") + except FileExistsError: + return + handle.close() + write_event(source, LIMIT_EVENT, {"max_session_bytes": limit}, session=session) + + def emit( source: str, event: str, @@ -176,9 +251,22 @@ def emit( run_key: str | None = None, ) -> None: """Fire-and-forget. Records nothing and reports nothing if anything goes wrong.""" - if not enabled(): + pol = policy.resolve(config.ROOT) + if not pol.enabled: return try: - write_event(source, event, attrs, session=session, run_key=run_key) + session_id = session or current_session_id() + root = trace_root() + session_dir = root / session_slug(session_id) + if not session_dir.exists(): + _enforce_retention(root, pol.keep_sessions, exclude=session_dir.name) + try: + size = (session_dir / TRACE_FILE).stat().st_size + except FileNotFoundError: + size = 0 + if size >= pol.max_session_bytes: + _mark_limit_once(session_dir, source, session_id, pol.max_session_bytes) + return + write_event(source, event, attrs, session=session_id, run_key=run_key) except Exception: # noqa: BLE001 - telemetry must never break the caller pass diff --git a/tools/chemenu/tests/conftest.py b/tools/chemenu/tests/conftest.py index 4f832b7..2f86d27 100644 --- a/tools/chemenu/tests/conftest.py +++ b/tools/chemenu/tests/conftest.py @@ -6,6 +6,7 @@ import pytest from chemenu import config, conventions from chemenu.frontmatter_io import write_page +from chemenu.telemetry import policy as telemetry_policy from chemenu.type_resolver import resolver # Environment the tool reads for its own behaviour. Cleared for every test, so @@ -17,6 +18,8 @@ _WIKITOOL_ENV = ( "WIKI_TRACE", "WIKI_TRACE_CONTENT", "WIKI_TRACE_MAX_CONTENT", + "WIKI_TRACE_MAX_SESSION_BYTES", + "WIKI_TRACE_KEEP_SESSIONS", "WIKITOOL_SESSION_ID", "WIKITOOL_UPDATE_URL", "WIKITOOL_UPDATE_TOKEN", @@ -152,15 +155,20 @@ def hermetic_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): # so neither a leak from before nor one from this test can be inherited. # One layer further in again: `conventions` parses `kb/CONVENTIONS.md` once # and keys the result on the file's own path and stat, so a repointed - # `KB_DIR` cannot be answered out of it. Cleared here anyway, on both sides, - # for the same reason `config.reset()` is - a fixture that leaves state - # behind is the hole this file exists to close, and the cost of proving it - # cannot leak is one function call per test. + # `KB_DIR` cannot be answered out of it. `telemetry.policy` caches the same + # way, keyed on the resolved root, for the release stamp and the telemetry + # config file it reads (not for `WIKI_TRACE` and its overrides, which stay + # live). Both cleared here anyway, on both sides, for the same reason + # `config.reset()` is - a fixture that leaves state behind is the hole + # this file exists to close, and the cost of proving it cannot leak is one + # function call per test. config.reset() conventions.reset_cache() + telemetry_policy.reset_cache() yield home config.reset() conventions.reset_cache() + telemetry_policy.reset_cache() def use_shipped_type_specs(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tools/chemenu/tests/test_doctor.py b/tools/chemenu/tests/test_doctor.py index 24e76c8..ce5bef6 100644 --- a/tools/chemenu/tests/test_doctor.py +++ b/tools/chemenu/tests/test_doctor.py @@ -345,6 +345,45 @@ def test_not_a_git_repo_fails(tmp_path, monkeypatch): assert _status(checks, "git-repo") == "FAIL" +def test_telemetry_is_on_by_default_for_a_dev_checkout(instance): + checks = doctor.run_doctor() + assert _status(checks, "telemetry") == "OK" + detail = next(c.detail for c in checks if c.name == "telemetry") + assert detail.startswith("on (") + assert "0/250 session(s)" in detail + + +def test_telemetry_is_off_by_default_for_a_stamped_distribution(instance): + (config.ROOT / ".wikitool-release.json").write_text( + '{"version": "0.1.0", "exported_at": "2026-08-29"}', encoding="utf-8" + ) + detail = next(c.detail for c in doctor.run_doctor() if c.name == "telemetry") + assert detail.startswith("off (") + assert ".wikitool-release.json present" in detail + + +def test_telemetry_never_fails_either_way(instance): + (config.ROOT / ".wikitool-release.json").write_text( + '{"version": "0.1.0", "exported_at": "2026-08-29"}', encoding="utf-8" + ) + assert _status(doctor.run_doctor(), "telemetry") == "OK" + + +def test_telemetry_reports_the_current_session_count_and_byte_total(instance, monkeypatch): + from chemenu.telemetry.writer import TRACE_FILE, trace_root + + trace_dir = config.ROOT / "traces" + monkeypatch.setenv("WIKI_TRACE_DIR", str(trace_dir)) + for name, content in (("a", "111"), ("b", "22222")): + session_dir = trace_root() / name + session_dir.mkdir(parents=True) + (session_dir / TRACE_FILE).write_text(content, encoding="utf-8") + + detail = next(c.detail for c in doctor.run_doctor() if c.name == "telemetry") + assert "2/250 session(s)" in detail + assert "8 byte(s)" in detail # "111" + "22222" + + def test_doctor_command_exits_nonzero_only_on_fail(instance, capsys): import typer diff --git a/tools/chemenu/tests/test_mcp_server.py b/tools/chemenu/tests/test_mcp_server.py index d0f0583..bc72a24 100644 --- a/tools/chemenu/tests/test_mcp_server.py +++ b/tools/chemenu/tests/test_mcp_server.py @@ -237,6 +237,25 @@ def test_tracing_outside_the_corpus_or_switched_off_is_accepted(corpus, tmp_path check_trace_destination(corpus) +def test_a_dev_checkout_like_corpus_refuses_to_start_with_no_override_at_all(corpus, monkeypatch): + """The `corpus` fixture carries no `.wikitool-release.json`, so the guard + checks the same policy `wikitool doctor` and the writer would: installation- + form default on, and the server must refuse even with nothing set.""" + monkeypatch.delenv("WIKI_TRACE", raising=False) + monkeypatch.delenv("WIKI_TRACE_DIR", raising=False) + with pytest.raises(TraceWouldWriteIntoCorpus): + check_trace_destination(corpus) + + +def test_a_stamped_distribution_starts_with_no_override_at_all(corpus, monkeypatch): + """A distributed instance defaults telemetry off - the server must not need + `WIKI_TRACE=0` set for it, unlike a dev checkout.""" + (corpus / ".wikitool-release.json").write_text("{}", encoding="utf-8") + monkeypatch.delenv("WIKI_TRACE", raising=False) + monkeypatch.delenv("WIKI_TRACE_DIR", raising=False) + check_trace_destination(corpus) # must not raise + + def test_only_the_two_chosen_transports_are_offered(): """`sse` is reachable through the SDK and deliberately not offered: it is the superseded remote transport, and building on it now only moves the diff --git a/tools/chemenu/tests/test_telemetry_emit.py b/tools/chemenu/tests/test_telemetry_emit.py index bbbf4ff..d152a37 100644 --- a/tools/chemenu/tests/test_telemetry_emit.py +++ b/tools/chemenu/tests/test_telemetry_emit.py @@ -225,6 +225,86 @@ def test_a_session_without_a_trace_reads_as_empty(monkeypatch, tmp_path): assert reader.read_trace("never-ran") == [] +# --- quantity caps --- + +def test_the_byte_cap_stops_writes_and_records_one_limit_event(monkeypatch, tmp_path): + monkeypatch.setenv("WIKI_TRACE_DIR", str(tmp_path)) + monkeypatch.setenv("WIKI_TRACE_MAX_SESSION_BYTES", "1") + emit_mod.emit("wikitool", "wikitool.call", {"command": "lint"}, session="cap1") + # The session header alone already exceeds a 1-byte cap, so the very next + # call must be the one that trips it. + emit_mod.emit("wikitool", "wikitool.call", {"command": "index"}, session="cap1") + emit_mod.emit("wikitool", "wikitool.call", {"command": "search"}, session="cap1") + + records = read_lines(tmp_path / "cap1" / "trace.jsonl") + events = [r["event"] for r in records] + assert events.count("telemetry.limit") == 1 + assert events[-1] == "telemetry.limit" + # At most one event beyond whatever fit under the cap. + assert events == ["session.start", "wikitool.call", "telemetry.limit"] + + +def test_the_limit_marker_file_elects_a_single_writer(monkeypatch, tmp_path): + """Two processes racing the cap must not both append the marker event - + the same single-writer trick the session header uses.""" + monkeypatch.setenv("WIKI_TRACE_DIR", str(tmp_path)) + session_dir = tmp_path / "cap2" + session_dir.mkdir() + (session_dir / "trace.jsonl").write_text("x" * 100, encoding="utf-8") + monkeypatch.setenv("WIKI_TRACE_MAX_SESSION_BYTES", "10") + + emit_mod._mark_limit_once(session_dir, "wikitool", "cap2", 10) + emit_mod._mark_limit_once(session_dir, "wikitool", "cap2", 10) # no-op, marker exists + + text = (session_dir / "trace.jsonl").read_text(encoding="utf-8") + assert text.count("telemetry.limit") == 1 + + +def test_retention_keeps_only_the_most_recent_sessions(monkeypatch, tmp_path): + monkeypatch.setenv("WIKI_TRACE_DIR", str(tmp_path)) + monkeypatch.setenv("WIKI_TRACE_KEEP_SESSIONS", "2") + import time + + for name in ("old", "middle", "new"): + emit_mod.emit("wikitool", "wikitool.call", session=name) + time.sleep(0.01) # distinct mtimes + + emit_mod.emit("wikitool", "wikitool.call", session="newest") + + remaining = {p.name for p in tmp_path.iterdir() if p.is_dir()} + assert remaining == {"new", "newest"} + assert not (tmp_path / "old" / "trace.jsonl").exists() + assert not (tmp_path / "old").exists() + + +def test_retention_only_runs_once_per_new_session_directory(monkeypatch, tmp_path): + """Repeated writes into an already-open session must not re-scan and must + never delete that session's own directory, however old it gets.""" + monkeypatch.setenv("WIKI_TRACE_DIR", str(tmp_path)) + monkeypatch.setenv("WIKI_TRACE_KEEP_SESSIONS", "1") + emit_mod.emit("wikitool", "wikitool.call", session="only") + for _ in range(5): + emit_mod.emit("wikitool", "wikitool.call", session="only") + assert (tmp_path / "only" / "trace.jsonl").exists() + + +def test_retention_leaves_a_foreign_file_and_its_directory_standing(monkeypatch, tmp_path): + """A retention pass deletes only trace.jsonl/.limit; a directory that still + holds something else after that is never rmdir'd.""" + monkeypatch.setenv("WIKI_TRACE_DIR", str(tmp_path)) + monkeypatch.setenv("WIKI_TRACE_KEEP_SESSIONS", "1") + stale = tmp_path / "stale" + stale.mkdir() + (stale / "trace.jsonl").write_text("{}\n", encoding="utf-8") + (stale / "saved.json").write_text("{}\n", encoding="utf-8") + + emit_mod.emit("wikitool", "wikitool.call", session="fresh") + + assert stale.exists() + assert (stale / "saved.json").exists() + assert not (stale / "trace.jsonl").exists() + + def test_completeness_is_the_union_across_sources(): records = [ {"event": "session.start", "attrs": {"completeness": ["tool.pre", "tool.post"]}}, diff --git a/tools/chemenu/tests/test_telemetry_policy.py b/tools/chemenu/tests/test_telemetry_policy.py new file mode 100644 index 0000000..68392b8 --- /dev/null +++ b/tools/chemenu/tests/test_telemetry_policy.py @@ -0,0 +1,127 @@ +"""Telemetry policy: installation-form default, the two overrides that beat +it, and the caching split between the filesystem-derived half and the +always-live environment half.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from chemenu.telemetry import policy + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + for var in (policy.ENV_ENABLED, policy.ENV_MAX_SESSION_BYTES, policy.ENV_KEEP_SESSIONS): + monkeypatch.delenv(var, raising=False) + policy.reset_cache() + yield + policy.reset_cache() + + +def _stamp(root: Path) -> None: + (root / ".wikitool-release.json").write_text("{}", encoding="utf-8") + + +def _config_file(root: Path, **fields) -> None: + import json + + (root / ".wikitool-telemetry.json").write_text(json.dumps(fields), encoding="utf-8") + + +def test_a_dev_checkout_defaults_on(tmp_path): + pol = policy.resolve(tmp_path) + assert pol.enabled is True + assert "dev checkout" in pol.reason + + +def test_a_stamped_distribution_defaults_off(tmp_path): + _stamp(tmp_path) + pol = policy.resolve(tmp_path) + assert pol.enabled is False + assert "distributed instance" in pol.reason + + +def test_the_config_file_overrides_the_form_default(tmp_path): + _stamp(tmp_path) + _config_file(tmp_path, enabled=True) + pol = policy.resolve(tmp_path) + assert pol.enabled is True + assert ".wikitool-telemetry.json" in pol.reason + + +def test_the_env_var_overrides_the_config_file_in_both_directions(tmp_path, monkeypatch): + _config_file(tmp_path, enabled=True) + monkeypatch.setenv(policy.ENV_ENABLED, "0") + assert policy.resolve(tmp_path).enabled is False + + _stamp(tmp_path) + _config_file(tmp_path, enabled=False) + monkeypatch.setenv(policy.ENV_ENABLED, "1") + assert policy.resolve(tmp_path).enabled is True + + +def test_default_caps(tmp_path): + pol = policy.resolve(tmp_path) + assert pol.max_session_bytes == policy.DEFAULT_MAX_SESSION_BYTES + assert pol.keep_sessions == policy.DEFAULT_KEEP_SESSIONS + + +def test_the_config_file_can_set_both_caps(tmp_path): + _config_file(tmp_path, max_session_bytes=1024, keep_sessions=5) + pol = policy.resolve(tmp_path) + assert pol.max_session_bytes == 1024 + assert pol.keep_sessions == 5 + + +def test_env_caps_override_the_config_file(tmp_path, monkeypatch): + _config_file(tmp_path, max_session_bytes=1024, keep_sessions=5) + monkeypatch.setenv(policy.ENV_MAX_SESSION_BYTES, "2048") + monkeypatch.setenv(policy.ENV_KEEP_SESSIONS, "9") + pol = policy.resolve(tmp_path) + assert pol.max_session_bytes == 2048 + assert pol.keep_sessions == 9 + + +def test_a_non_positive_or_unparsable_override_is_ignored(tmp_path, monkeypatch): + monkeypatch.setenv(policy.ENV_MAX_SESSION_BYTES, "0") + monkeypatch.setenv(policy.ENV_KEEP_SESSIONS, "not-a-number") + pol = policy.resolve(tmp_path) + assert pol.max_session_bytes == policy.DEFAULT_MAX_SESSION_BYTES + assert pol.keep_sessions == policy.DEFAULT_KEEP_SESSIONS + + +def test_a_malformed_config_file_is_treated_as_absent(tmp_path): + (tmp_path / ".wikitool-telemetry.json").write_text("{not json", encoding="utf-8") + pol = policy.resolve(tmp_path) + assert pol.enabled is True # falls through to the dev-checkout form default + + +def test_the_filesystem_half_is_cached_per_root(tmp_path): + """Writing the stamp *after* the first resolve must not retroactively + change the cached answer - `reset_cache()` is required to see it.""" + first = policy.resolve(tmp_path) + assert first.enabled is True + _stamp(tmp_path) + still_cached = policy.resolve(tmp_path) + assert still_cached.enabled is True + policy.reset_cache() + assert policy.resolve(tmp_path).enabled is False + + +def test_the_env_override_is_always_live_even_with_a_cached_root(tmp_path, monkeypatch): + """The env var must never need a `reset_cache()` to take effect - only the + filesystem-derived half is cached.""" + assert policy.resolve(tmp_path).enabled is True + monkeypatch.setenv(policy.ENV_ENABLED, "0") + assert policy.resolve(tmp_path).enabled is False + monkeypatch.delenv(policy.ENV_ENABLED) + assert policy.resolve(tmp_path).enabled is True + + +def test_resolve_defaults_to_config_root(tmp_path, monkeypatch): + from chemenu import config + + monkeypatch.setattr(config, "ROOT", tmp_path) + _stamp(tmp_path) + assert policy.resolve().enabled is False diff --git a/tools/chemenu/version.py b/tools/chemenu/version.py index c2ff734..dffd689 100644 --- a/tools/chemenu/version.py +++ b/tools/chemenu/version.py @@ -51,7 +51,8 @@ CHANGES_FILENAME = "CHANGES.md" # Written into an exported distribution by `dist export`, and committed with # it: an instance has to keep knowing which stack it came from, or update # detection has nothing to compare against. Machine-written, never hand-edited. -RELEASE_STAMP_FILENAME = ".wikitool-release.json" +# Defined in `config.py`, not here - see that constant's own comment. +RELEASE_STAMP_FILENAME = config.RELEASE_STAMP_FILENAME STAMP_SCHEMA = 1 # Where `version check` looks when neither the environment nor a release stamp