stack: Telemetrie-Default nach Installationsform, Byte-Deckel und Session-Retention (schliesst #55)
Files changed: - .gitea/workflows/ci.yml - .gitignore - CHANGES.md - EVALS.md - INSTALL-MCP.md - INSTALL.md - VERSION - instructions/setup-instance.md - reports/CONTRACT.md - tools/CONTRACT.md - tools/chemenu/commands/doctor.py - tools/chemenu/config.py - tools/chemenu/mcp/server.py - tools/chemenu/telemetry/policy.py - tools/chemenu/telemetry/schema.py - tools/chemenu/telemetry/writer.py - tools/chemenu/tests/conftest.py - tools/chemenu/tests/test_doctor.py - tools/chemenu/tests/test_mcp_server.py - tools/chemenu/tests/test_telemetry_emit.py - tools/chemenu/tests/test_telemetry_policy.py - tools/chemenu/version.py
This commit is contained in:
@@ -345,6 +345,37 @@ def check_generated_files() -> Check:
|
||||
return Check("generated-files", "OK", "kb/index.md, kb/log.md, kb/provenance.md present")
|
||||
|
||||
|
||||
def check_telemetry() -> Check:
|
||||
"""Whether tracing is on for this checkout, why, and how full its two caps
|
||||
are.
|
||||
|
||||
Never `FAIL`s, the same line `check_publish_remotes` and
|
||||
`check_environment` draw: both an enabled and a disabled tree are
|
||||
legitimate states, and a `FAIL` would make one of them mandatory by the
|
||||
back door. The tree walk below is fine here - `doctor` is not a hot path,
|
||||
unlike the `stat` `emit()` does on every append.
|
||||
"""
|
||||
from chemenu.telemetry import policy as telemetry_policy
|
||||
from chemenu.telemetry import reader
|
||||
from chemenu.telemetry.writer import TRACE_FILE, trace_root
|
||||
|
||||
pol = telemetry_policy.resolve(config.ROOT)
|
||||
root = trace_root()
|
||||
session_count = len(reader.sessions(root))
|
||||
total_bytes = (
|
||||
sum(f.stat().st_size for f in root.glob(f"*/{TRACE_FILE}") if f.is_file())
|
||||
if root.exists()
|
||||
else 0
|
||||
)
|
||||
state = "on" if pol.enabled else "off"
|
||||
return Check(
|
||||
"telemetry", "OK",
|
||||
f"{state} ({pol.reason}); {session_count}/{pol.keep_sessions} session(s), "
|
||||
f"{total_bytes:,} byte(s) under {rel_path(root)} "
|
||||
f"(cap {pol.max_session_bytes:,} byte(s)/session)",
|
||||
)
|
||||
|
||||
|
||||
def check_session_id() -> Check:
|
||||
import os
|
||||
|
||||
@@ -449,6 +480,7 @@ def run_doctor() -> list[Check]:
|
||||
check_publish_remotes(),
|
||||
check_generated_files(),
|
||||
check_session_id(),
|
||||
check_telemetry(),
|
||||
]
|
||||
return checks
|
||||
|
||||
@@ -458,7 +490,8 @@ def doctor_command(
|
||||
):
|
||||
"""Check that this instance is correctly configured: dependencies, author,
|
||||
git identity/remote, published skills, structure, personalization, KB
|
||||
conventions, generated files, and session scoping. Read-only. Exits 1 only
|
||||
conventions, generated files, session scoping, and telemetry state.
|
||||
Read-only. Exits 1 only
|
||||
if a check FAILs."""
|
||||
checks = run_doctor()
|
||||
|
||||
|
||||
@@ -202,6 +202,24 @@ ENVIRONMENT_TEMPLATE = f"{ENVIRONMENT_FILE}.template"
|
||||
# `dist export` already computes (AGENTS.md invariant 8). See NOTICE.
|
||||
LICENSE_FILES = ("LICENSE", "LICENSE-CONTENT", "NOTICE")
|
||||
|
||||
# Written by `dist export` into every distribution; presence means "this tree
|
||||
# is an exported instance", absence means "this is the dev checkout the stack
|
||||
# ships from". Defined here rather than only in `version.py`, because
|
||||
# `chemenu.telemetry` (`telemetry/policy.py`) needs the same marker to answer
|
||||
# opt-in vs. opt-out and must stay importable without the venv - a hook
|
||||
# handler imports it on every tool call. `version.py` re-exports this name
|
||||
# rather than defining its own, per AGENTS.md invariant 8.
|
||||
RELEASE_STAMP_FILENAME = ".wikitool-release.json"
|
||||
|
||||
# Per-checkout telemetry opt-in/opt-out plus its two quantity caps (see
|
||||
# `telemetry/policy.py`). Same shape as `PUBLISH_REMOTES_FILENAME` below: it
|
||||
# answers a question about *this* checkout, so it is per-checkout and
|
||||
# gitignored, ships no `.template`, and its absence is a legitimate state -
|
||||
# the installation-form default (keyed off `RELEASE_STAMP_FILENAME`) applies.
|
||||
# `instructions/setup-instance.md`'s Telemetry decision point writes it from
|
||||
# the operator's answer.
|
||||
TELEMETRY_FILENAME = ".wikitool-telemetry.json"
|
||||
|
||||
# Which push targets `publish` may write to, for a checkout that says so. The
|
||||
# danger this addresses is one checkout's content reaching another checkout's
|
||||
# remote - a private instance pushing its own `kb/` to a public upstream, where
|
||||
|
||||
@@ -43,6 +43,7 @@ from mcp.server.mcpserver.exceptions import ToolError
|
||||
from chemenu import config
|
||||
from chemenu.api import Corpus
|
||||
from chemenu.errors import ChemenuError
|
||||
from chemenu.telemetry import policy
|
||||
|
||||
SERVER_NAME = "chemenu"
|
||||
|
||||
@@ -62,21 +63,24 @@ class TraceWouldWriteIntoCorpus(RuntimeError):
|
||||
def check_trace_destination(root: Path) -> None:
|
||||
"""Refuse to start if a trace would be written into the corpus.
|
||||
|
||||
Telemetry defaults to *on* and writes under `reports/telemetry/` in the
|
||||
repo. Today nothing on this path emits - the writer is wired into
|
||||
`cli.main()` and the two gates, none of which run here - so this is a guard
|
||||
against the future rather than a fix for the present. It is worth having
|
||||
anyway: the sync that keeps this checkout current is `git reset --hard`, so
|
||||
a trace written into the tree is both a per-request write into a directory
|
||||
something else is entitled to wipe, and a silent way for the server to
|
||||
dirty the tree its own cache keys on.
|
||||
Telemetry's own default now depends on the installation form
|
||||
(`chemenu.telemetry.policy`) - off for a distributed instance, on for a
|
||||
dev checkout - so this guard resolves the same policy rather than reading
|
||||
`WIKI_TRACE` a second time, which is how it used to drift from the writer
|
||||
(AGENTS.md invariant 8). Today nothing on this path emits - the writer is
|
||||
wired into `cli.main()` and the two gates, none of which run here - so
|
||||
this is a guard against the future rather than a fix for the present. It
|
||||
is worth having anyway: the sync that keeps this checkout current is
|
||||
`git reset --hard`, so a trace written into the tree is both a per-request
|
||||
write into a directory something else is entitled to wipe, and a silent
|
||||
way for the server to dirty the tree its own cache keys on.
|
||||
|
||||
Turn tracing off (`WIKI_TRACE=0`) or point it somewhere else
|
||||
(`WIKI_TRACE_DIR`). Refusing rather than correcting it: a server that
|
||||
quietly relocates the operator's telemetry is a surprise waiting in a log
|
||||
nobody reads.
|
||||
"""
|
||||
if os.environ.get("WIKI_TRACE", "1") == "0":
|
||||
if not policy.resolve(root).enabled:
|
||||
return
|
||||
destination = os.environ.get("WIKI_TRACE_DIR")
|
||||
if destination is None:
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Telemetry on/off and its two quantity caps, resolved once per root.
|
||||
|
||||
One place to answer three questions, because `writer.py`, `wikitool doctor`
|
||||
and the MCP server's start-up guard must agree on the same answer for the
|
||||
same checkout - a second copy of this logic is exactly how the MCP server
|
||||
used to drift from the writer (it read `WIKI_TRACE` itself; fixed alongside
|
||||
this module, see AGENTS.md invariant 8).
|
||||
|
||||
**Precedence for `enabled`:** `WIKI_TRACE` (either direction) beats
|
||||
`.wikitool-telemetry.json`, which beats the installation-form default. The
|
||||
form is read off `config.RELEASE_STAMP_FILENAME`: present means an exported/
|
||||
distributed tree (default *off* - an operator never asked for telemetry),
|
||||
absent means the dev checkout this package ships from (default *on* - the
|
||||
traces are this stack's own measuring instrument, see EVALS.md).
|
||||
|
||||
**Caching.** Only the filesystem-derived half - the release stamp and the
|
||||
config file - is cached, keyed on the resolved root. `WIKI_TRACE` and the two
|
||||
`WIKI_TRACE_MAX_SESSION_BYTES`/`WIKI_TRACE_KEEP_SESSIONS` overrides are read
|
||||
fresh on every call: they are cheap (no I/O) and a test that flips one
|
||||
mid-run must see the new value immediately, not a cached one. A test that
|
||||
rewrites the release stamp or the config file at an already-resolved root
|
||||
calls `reset_cache()` itself, the same discipline `conventions.reset_cache()`
|
||||
already follows.
|
||||
|
||||
Stdlib only, like the rest of `chemenu.telemetry` - a hook handler imports
|
||||
this package on every tool call and must not need the venv.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from chemenu import config
|
||||
|
||||
DEFAULT_MAX_SESSION_BYTES = 5 * 1024 * 1024
|
||||
DEFAULT_KEEP_SESSIONS = 250
|
||||
|
||||
ENV_ENABLED = "WIKI_TRACE"
|
||||
ENV_MAX_SESSION_BYTES = "WIKI_TRACE_MAX_SESSION_BYTES"
|
||||
ENV_KEEP_SESSIONS = "WIKI_TRACE_KEEP_SESSIONS"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Policy:
|
||||
enabled: bool
|
||||
reason: str # human-readable - what decided `enabled`, for `doctor`
|
||||
max_session_bytes: int
|
||||
keep_sessions: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _RootDefaults:
|
||||
form_default: bool
|
||||
form_reason: str
|
||||
file_enabled: bool | None
|
||||
file_reason: str | None
|
||||
file_max_bytes: int | None
|
||||
file_keep_sessions: int | None
|
||||
|
||||
|
||||
_cache: dict[Path, _RootDefaults] = {}
|
||||
|
||||
|
||||
def reset_cache() -> None:
|
||||
_cache.clear()
|
||||
|
||||
|
||||
def _positive_int(value: object) -> int | None:
|
||||
try:
|
||||
parsed = int(value) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return parsed if parsed > 0 else None
|
||||
|
||||
|
||||
def _read_config(root: Path) -> dict:
|
||||
try:
|
||||
raw = (root / config.TELEMETRY_FILENAME).read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _root_defaults(root: Path) -> _RootDefaults:
|
||||
cached = _cache.get(root)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
distribution = (root / config.RELEASE_STAMP_FILENAME).is_file()
|
||||
form_default = not distribution
|
||||
form_reason = (
|
||||
f"no {config.RELEASE_STAMP_FILENAME} - dev checkout, default on"
|
||||
if form_default
|
||||
else f"{config.RELEASE_STAMP_FILENAME} present - distributed instance, default off"
|
||||
)
|
||||
|
||||
data = _read_config(root)
|
||||
file_enabled = bool(data["enabled"]) if "enabled" in data else None
|
||||
file_reason = (
|
||||
f"{config.TELEMETRY_FILENAME} sets enabled={file_enabled}"
|
||||
if file_enabled is not None
|
||||
else None
|
||||
)
|
||||
|
||||
result = _RootDefaults(
|
||||
form_default=form_default,
|
||||
form_reason=form_reason,
|
||||
file_enabled=file_enabled,
|
||||
file_reason=file_reason,
|
||||
file_max_bytes=_positive_int(data.get("max_session_bytes")),
|
||||
file_keep_sessions=_positive_int(data.get("keep_sessions")),
|
||||
)
|
||||
_cache[root] = result
|
||||
return result
|
||||
|
||||
|
||||
def resolve(root: "Path | str | None" = None) -> Policy:
|
||||
"""The effective policy for `root` (default: `config.ROOT`)."""
|
||||
resolved_root = Path(root).resolve() if root is not None else Path(config.ROOT).resolve()
|
||||
defaults = _root_defaults(resolved_root)
|
||||
|
||||
env_enabled = os.environ.get(ENV_ENABLED)
|
||||
if env_enabled is not None:
|
||||
enabled = env_enabled != "0"
|
||||
reason = f"{ENV_ENABLED}={env_enabled!r} overrides"
|
||||
elif defaults.file_enabled is not None:
|
||||
enabled = defaults.file_enabled
|
||||
reason = defaults.file_reason # type: ignore[assignment]
|
||||
else:
|
||||
enabled = defaults.form_default
|
||||
reason = defaults.form_reason
|
||||
|
||||
max_bytes = (
|
||||
_positive_int(os.environ.get(ENV_MAX_SESSION_BYTES))
|
||||
or defaults.file_max_bytes
|
||||
or DEFAULT_MAX_SESSION_BYTES
|
||||
)
|
||||
keep_sessions = (
|
||||
_positive_int(os.environ.get(ENV_KEEP_SESSIONS))
|
||||
or defaults.file_keep_sessions
|
||||
or DEFAULT_KEEP_SESSIONS
|
||||
)
|
||||
|
||||
return Policy(
|
||||
enabled=enabled,
|
||||
reason=reason,
|
||||
max_session_bytes=max_bytes,
|
||||
keep_sessions=keep_sessions,
|
||||
)
|
||||
@@ -57,6 +57,7 @@ OPTIONAL_EVENTS = frozenset(
|
||||
"publish.commit",
|
||||
"budget.state",
|
||||
"gate.cleared",
|
||||
"telemetry.limit",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -1,17 +1,26 @@
|
||||
"""Appends trace events to `reports/telemetry/<session>/trace.jsonl`.
|
||||
|
||||
Two rules govern this module.
|
||||
Three rules govern this module.
|
||||
|
||||
**Telemetry never breaks the tool.** `emit()` swallows everything: a full disk,
|
||||
a permission error or a bug in a scrubber pattern must not turn a working
|
||||
`wikitool` command - or a hook wrapped around someone's tool call - into a
|
||||
failure. Tests call `write_event()` instead, which raises.
|
||||
failure. Tests call `write_event()` instead, which raises - and, unlike
|
||||
`emit()`, bypasses policy entirely: no `enabled()` check, no byte cap, no
|
||||
retention. A test wanting those exercises `emit()`.
|
||||
|
||||
**Append, do not rewrite.** The budget state is a whole-file document and is
|
||||
written with the temp-file + `os.replace` dance. A trace is append-only, so the
|
||||
equivalent guarantee is `O_APPEND` plus an exclusive lock: several processes
|
||||
write to one trace at once (the CLI in one, a hook handler per tool call in
|
||||
another), and a line must never land inside another line.
|
||||
|
||||
**Two quantity caps, both enforced in `emit()`, both fail-silent.** A byte cap
|
||||
per session trace (`policy.max_session_bytes()`), checked with one `stat`
|
||||
before every append; and a retention pass over `trace_root()`'s
|
||||
session directories, run only when a brand-new one is about to be created -
|
||||
never per event. See `chemenu.telemetry.policy` for where the numbers come
|
||||
from.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -22,18 +31,26 @@ from pathlib import Path
|
||||
from chemenu import config
|
||||
from chemenu.session import session_id as current_session_id
|
||||
from chemenu.session import session_slug
|
||||
from chemenu.telemetry import schema, scrub
|
||||
from chemenu.telemetry import policy, schema, scrub
|
||||
|
||||
TRACE_ROOT = config.REPORTS_DIR / "telemetry"
|
||||
TRACE_FILE = "trace.jsonl"
|
||||
|
||||
# The single-writer sentinel that elects which process records the
|
||||
# `telemetry.limit` marker for a session that has hit its byte cap - see
|
||||
# `_mark_limit_once`. Also what `_enforce_retention` deletes alongside
|
||||
# `trace.jsonl`: these two names are everything this policy owns per session
|
||||
# directory.
|
||||
LIMIT_MARKER = ".limit"
|
||||
LIMIT_EVENT = "telemetry.limit"
|
||||
|
||||
# Per-process counter. Not comparable across processes - see schema.py on how
|
||||
# to order a trace.
|
||||
_seq = 0
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
return os.environ.get("WIKI_TRACE", "1") != "0"
|
||||
return policy.resolve(config.ROOT).enabled
|
||||
|
||||
|
||||
def trace_root() -> Path:
|
||||
@@ -167,6 +184,64 @@ def _locked_write(handle, line: str) -> None:
|
||||
fcntl.flock(handle, fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def _enforce_retention(root: Path, keep: int, exclude: str) -> None:
|
||||
"""Delete the oldest session directories' `trace.jsonl`/`.limit` so that,
|
||||
once the session about to be created lands, at most `keep` remain.
|
||||
|
||||
Runs once, right before a brand-new session directory would be created -
|
||||
never per event, so an active session is never re-scanned for its own
|
||||
writes. It reserves that new session's own slot up front (keeping
|
||||
`keep - 1` of what already exists) rather than trimming to `keep` and then
|
||||
letting the new one land as `keep + 1`: the latter never actually
|
||||
converges back to `keep` under a steady trickle of new sessions, one at a
|
||||
time, each pass only ever removing what the *previous* pass left over the
|
||||
limit. Sorted by `mtime`, most recent first, the same order
|
||||
`reader.sessions()` reads a trace root back in, so "oldest" here and
|
||||
"most recent" there agree.
|
||||
|
||||
`exclude` is the session about to be created - it does not exist on disk
|
||||
yet in the common case, but is named explicitly anyway so a session whose
|
||||
directory a concurrent process just created is never the one this pass
|
||||
removes, regardless of its mtime.
|
||||
|
||||
Deletes only the two files this policy owns per session; the directory
|
||||
itself is `rmdir`-ed only once empty, never `rmtree`-d. A foreign file
|
||||
left by something else in a session's directory keeps that directory, and
|
||||
everything else in it, standing - `reports/` holds local, non-recomputable
|
||||
data no retention pass has business deleting wholesale.
|
||||
"""
|
||||
if not root.is_dir():
|
||||
return
|
||||
candidates = [p for p in root.iterdir() if p.is_dir() and p.name != exclude]
|
||||
candidates.sort(key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
for stale in candidates[max(keep - 1, 0):]:
|
||||
for name in (TRACE_FILE, LIMIT_MARKER):
|
||||
try:
|
||||
(stale / name).unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
try:
|
||||
stale.rmdir()
|
||||
except OSError:
|
||||
pass # not empty - something else lives here, leave it standing
|
||||
|
||||
|
||||
def _mark_limit_once(session_dir: Path, source: str, session: str, limit: int) -> None:
|
||||
"""Record `telemetry.limit` exactly once per session, however many further
|
||||
calls hit the cap afterwards.
|
||||
|
||||
The single-writer trick is the same one `_seed_session_header` uses:
|
||||
several processes can hit the cap on the same trace at once, and an
|
||||
`exists()` check would let more than one of them win.
|
||||
"""
|
||||
try:
|
||||
handle = open(session_dir / LIMIT_MARKER, "x", encoding="utf-8")
|
||||
except FileExistsError:
|
||||
return
|
||||
handle.close()
|
||||
write_event(source, LIMIT_EVENT, {"max_session_bytes": limit}, session=session)
|
||||
|
||||
|
||||
def emit(
|
||||
source: str,
|
||||
event: str,
|
||||
@@ -176,9 +251,22 @@ def emit(
|
||||
run_key: str | None = None,
|
||||
) -> None:
|
||||
"""Fire-and-forget. Records nothing and reports nothing if anything goes wrong."""
|
||||
if not enabled():
|
||||
pol = policy.resolve(config.ROOT)
|
||||
if not pol.enabled:
|
||||
return
|
||||
try:
|
||||
write_event(source, event, attrs, session=session, run_key=run_key)
|
||||
session_id = session or current_session_id()
|
||||
root = trace_root()
|
||||
session_dir = root / session_slug(session_id)
|
||||
if not session_dir.exists():
|
||||
_enforce_retention(root, pol.keep_sessions, exclude=session_dir.name)
|
||||
try:
|
||||
size = (session_dir / TRACE_FILE).stat().st_size
|
||||
except FileNotFoundError:
|
||||
size = 0
|
||||
if size >= pol.max_session_bytes:
|
||||
_mark_limit_once(session_dir, source, session_id, pol.max_session_bytes)
|
||||
return
|
||||
write_event(source, event, attrs, session=session_id, run_key=run_key)
|
||||
except Exception: # noqa: BLE001 - telemetry must never break the caller
|
||||
pass
|
||||
|
||||
@@ -6,6 +6,7 @@ import pytest
|
||||
|
||||
from chemenu import config, conventions
|
||||
from chemenu.frontmatter_io import write_page
|
||||
from chemenu.telemetry import policy as telemetry_policy
|
||||
from chemenu.type_resolver import resolver
|
||||
|
||||
# Environment the tool reads for its own behaviour. Cleared for every test, so
|
||||
@@ -17,6 +18,8 @@ _WIKITOOL_ENV = (
|
||||
"WIKI_TRACE",
|
||||
"WIKI_TRACE_CONTENT",
|
||||
"WIKI_TRACE_MAX_CONTENT",
|
||||
"WIKI_TRACE_MAX_SESSION_BYTES",
|
||||
"WIKI_TRACE_KEEP_SESSIONS",
|
||||
"WIKITOOL_SESSION_ID",
|
||||
"WIKITOOL_UPDATE_URL",
|
||||
"WIKITOOL_UPDATE_TOKEN",
|
||||
@@ -152,15 +155,20 @@ def hermetic_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
# so neither a leak from before nor one from this test can be inherited.
|
||||
# One layer further in again: `conventions` parses `kb/CONVENTIONS.md` once
|
||||
# and keys the result on the file's own path and stat, so a repointed
|
||||
# `KB_DIR` cannot be answered out of it. Cleared here anyway, on both sides,
|
||||
# for the same reason `config.reset()` is - a fixture that leaves state
|
||||
# behind is the hole this file exists to close, and the cost of proving it
|
||||
# cannot leak is one function call per test.
|
||||
# `KB_DIR` cannot be answered out of it. `telemetry.policy` caches the same
|
||||
# way, keyed on the resolved root, for the release stamp and the telemetry
|
||||
# config file it reads (not for `WIKI_TRACE` and its overrides, which stay
|
||||
# live). Both cleared here anyway, on both sides, for the same reason
|
||||
# `config.reset()` is - a fixture that leaves state behind is the hole
|
||||
# this file exists to close, and the cost of proving it cannot leak is one
|
||||
# function call per test.
|
||||
config.reset()
|
||||
conventions.reset_cache()
|
||||
telemetry_policy.reset_cache()
|
||||
yield home
|
||||
config.reset()
|
||||
conventions.reset_cache()
|
||||
telemetry_policy.reset_cache()
|
||||
|
||||
|
||||
def use_shipped_type_specs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
||||
@@ -345,6 +345,45 @@ def test_not_a_git_repo_fails(tmp_path, monkeypatch):
|
||||
assert _status(checks, "git-repo") == "FAIL"
|
||||
|
||||
|
||||
def test_telemetry_is_on_by_default_for_a_dev_checkout(instance):
|
||||
checks = doctor.run_doctor()
|
||||
assert _status(checks, "telemetry") == "OK"
|
||||
detail = next(c.detail for c in checks if c.name == "telemetry")
|
||||
assert detail.startswith("on (")
|
||||
assert "0/250 session(s)" in detail
|
||||
|
||||
|
||||
def test_telemetry_is_off_by_default_for_a_stamped_distribution(instance):
|
||||
(config.ROOT / ".wikitool-release.json").write_text(
|
||||
'{"version": "0.1.0", "exported_at": "2026-08-29"}', encoding="utf-8"
|
||||
)
|
||||
detail = next(c.detail for c in doctor.run_doctor() if c.name == "telemetry")
|
||||
assert detail.startswith("off (")
|
||||
assert ".wikitool-release.json present" in detail
|
||||
|
||||
|
||||
def test_telemetry_never_fails_either_way(instance):
|
||||
(config.ROOT / ".wikitool-release.json").write_text(
|
||||
'{"version": "0.1.0", "exported_at": "2026-08-29"}', encoding="utf-8"
|
||||
)
|
||||
assert _status(doctor.run_doctor(), "telemetry") == "OK"
|
||||
|
||||
|
||||
def test_telemetry_reports_the_current_session_count_and_byte_total(instance, monkeypatch):
|
||||
from chemenu.telemetry.writer import TRACE_FILE, trace_root
|
||||
|
||||
trace_dir = config.ROOT / "traces"
|
||||
monkeypatch.setenv("WIKI_TRACE_DIR", str(trace_dir))
|
||||
for name, content in (("a", "111"), ("b", "22222")):
|
||||
session_dir = trace_root() / name
|
||||
session_dir.mkdir(parents=True)
|
||||
(session_dir / TRACE_FILE).write_text(content, encoding="utf-8")
|
||||
|
||||
detail = next(c.detail for c in doctor.run_doctor() if c.name == "telemetry")
|
||||
assert "2/250 session(s)" in detail
|
||||
assert "8 byte(s)" in detail # "111" + "22222"
|
||||
|
||||
|
||||
def test_doctor_command_exits_nonzero_only_on_fail(instance, capsys):
|
||||
import typer
|
||||
|
||||
|
||||
@@ -237,6 +237,25 @@ def test_tracing_outside_the_corpus_or_switched_off_is_accepted(corpus, tmp_path
|
||||
check_trace_destination(corpus)
|
||||
|
||||
|
||||
def test_a_dev_checkout_like_corpus_refuses_to_start_with_no_override_at_all(corpus, monkeypatch):
|
||||
"""The `corpus` fixture carries no `.wikitool-release.json`, so the guard
|
||||
checks the same policy `wikitool doctor` and the writer would: installation-
|
||||
form default on, and the server must refuse even with nothing set."""
|
||||
monkeypatch.delenv("WIKI_TRACE", raising=False)
|
||||
monkeypatch.delenv("WIKI_TRACE_DIR", raising=False)
|
||||
with pytest.raises(TraceWouldWriteIntoCorpus):
|
||||
check_trace_destination(corpus)
|
||||
|
||||
|
||||
def test_a_stamped_distribution_starts_with_no_override_at_all(corpus, monkeypatch):
|
||||
"""A distributed instance defaults telemetry off - the server must not need
|
||||
`WIKI_TRACE=0` set for it, unlike a dev checkout."""
|
||||
(corpus / ".wikitool-release.json").write_text("{}", encoding="utf-8")
|
||||
monkeypatch.delenv("WIKI_TRACE", raising=False)
|
||||
monkeypatch.delenv("WIKI_TRACE_DIR", raising=False)
|
||||
check_trace_destination(corpus) # must not raise
|
||||
|
||||
|
||||
def test_only_the_two_chosen_transports_are_offered():
|
||||
"""`sse` is reachable through the SDK and deliberately not offered: it is
|
||||
the superseded remote transport, and building on it now only moves the
|
||||
|
||||
@@ -225,6 +225,86 @@ def test_a_session_without_a_trace_reads_as_empty(monkeypatch, tmp_path):
|
||||
assert reader.read_trace("never-ran") == []
|
||||
|
||||
|
||||
# --- quantity caps ---
|
||||
|
||||
def test_the_byte_cap_stops_writes_and_records_one_limit_event(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("WIKI_TRACE_DIR", str(tmp_path))
|
||||
monkeypatch.setenv("WIKI_TRACE_MAX_SESSION_BYTES", "1")
|
||||
emit_mod.emit("wikitool", "wikitool.call", {"command": "lint"}, session="cap1")
|
||||
# The session header alone already exceeds a 1-byte cap, so the very next
|
||||
# call must be the one that trips it.
|
||||
emit_mod.emit("wikitool", "wikitool.call", {"command": "index"}, session="cap1")
|
||||
emit_mod.emit("wikitool", "wikitool.call", {"command": "search"}, session="cap1")
|
||||
|
||||
records = read_lines(tmp_path / "cap1" / "trace.jsonl")
|
||||
events = [r["event"] for r in records]
|
||||
assert events.count("telemetry.limit") == 1
|
||||
assert events[-1] == "telemetry.limit"
|
||||
# At most one event beyond whatever fit under the cap.
|
||||
assert events == ["session.start", "wikitool.call", "telemetry.limit"]
|
||||
|
||||
|
||||
def test_the_limit_marker_file_elects_a_single_writer(monkeypatch, tmp_path):
|
||||
"""Two processes racing the cap must not both append the marker event -
|
||||
the same single-writer trick the session header uses."""
|
||||
monkeypatch.setenv("WIKI_TRACE_DIR", str(tmp_path))
|
||||
session_dir = tmp_path / "cap2"
|
||||
session_dir.mkdir()
|
||||
(session_dir / "trace.jsonl").write_text("x" * 100, encoding="utf-8")
|
||||
monkeypatch.setenv("WIKI_TRACE_MAX_SESSION_BYTES", "10")
|
||||
|
||||
emit_mod._mark_limit_once(session_dir, "wikitool", "cap2", 10)
|
||||
emit_mod._mark_limit_once(session_dir, "wikitool", "cap2", 10) # no-op, marker exists
|
||||
|
||||
text = (session_dir / "trace.jsonl").read_text(encoding="utf-8")
|
||||
assert text.count("telemetry.limit") == 1
|
||||
|
||||
|
||||
def test_retention_keeps_only_the_most_recent_sessions(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("WIKI_TRACE_DIR", str(tmp_path))
|
||||
monkeypatch.setenv("WIKI_TRACE_KEEP_SESSIONS", "2")
|
||||
import time
|
||||
|
||||
for name in ("old", "middle", "new"):
|
||||
emit_mod.emit("wikitool", "wikitool.call", session=name)
|
||||
time.sleep(0.01) # distinct mtimes
|
||||
|
||||
emit_mod.emit("wikitool", "wikitool.call", session="newest")
|
||||
|
||||
remaining = {p.name for p in tmp_path.iterdir() if p.is_dir()}
|
||||
assert remaining == {"new", "newest"}
|
||||
assert not (tmp_path / "old" / "trace.jsonl").exists()
|
||||
assert not (tmp_path / "old").exists()
|
||||
|
||||
|
||||
def test_retention_only_runs_once_per_new_session_directory(monkeypatch, tmp_path):
|
||||
"""Repeated writes into an already-open session must not re-scan and must
|
||||
never delete that session's own directory, however old it gets."""
|
||||
monkeypatch.setenv("WIKI_TRACE_DIR", str(tmp_path))
|
||||
monkeypatch.setenv("WIKI_TRACE_KEEP_SESSIONS", "1")
|
||||
emit_mod.emit("wikitool", "wikitool.call", session="only")
|
||||
for _ in range(5):
|
||||
emit_mod.emit("wikitool", "wikitool.call", session="only")
|
||||
assert (tmp_path / "only" / "trace.jsonl").exists()
|
||||
|
||||
|
||||
def test_retention_leaves_a_foreign_file_and_its_directory_standing(monkeypatch, tmp_path):
|
||||
"""A retention pass deletes only trace.jsonl/.limit; a directory that still
|
||||
holds something else after that is never rmdir'd."""
|
||||
monkeypatch.setenv("WIKI_TRACE_DIR", str(tmp_path))
|
||||
monkeypatch.setenv("WIKI_TRACE_KEEP_SESSIONS", "1")
|
||||
stale = tmp_path / "stale"
|
||||
stale.mkdir()
|
||||
(stale / "trace.jsonl").write_text("{}\n", encoding="utf-8")
|
||||
(stale / "saved.json").write_text("{}\n", encoding="utf-8")
|
||||
|
||||
emit_mod.emit("wikitool", "wikitool.call", session="fresh")
|
||||
|
||||
assert stale.exists()
|
||||
assert (stale / "saved.json").exists()
|
||||
assert not (stale / "trace.jsonl").exists()
|
||||
|
||||
|
||||
def test_completeness_is_the_union_across_sources():
|
||||
records = [
|
||||
{"event": "session.start", "attrs": {"completeness": ["tool.pre", "tool.post"]}},
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Telemetry policy: installation-form default, the two overrides that beat
|
||||
it, and the caching split between the filesystem-derived half and the
|
||||
always-live environment half."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from chemenu.telemetry import policy
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_env(monkeypatch):
|
||||
for var in (policy.ENV_ENABLED, policy.ENV_MAX_SESSION_BYTES, policy.ENV_KEEP_SESSIONS):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
policy.reset_cache()
|
||||
yield
|
||||
policy.reset_cache()
|
||||
|
||||
|
||||
def _stamp(root: Path) -> None:
|
||||
(root / ".wikitool-release.json").write_text("{}", encoding="utf-8")
|
||||
|
||||
|
||||
def _config_file(root: Path, **fields) -> None:
|
||||
import json
|
||||
|
||||
(root / ".wikitool-telemetry.json").write_text(json.dumps(fields), encoding="utf-8")
|
||||
|
||||
|
||||
def test_a_dev_checkout_defaults_on(tmp_path):
|
||||
pol = policy.resolve(tmp_path)
|
||||
assert pol.enabled is True
|
||||
assert "dev checkout" in pol.reason
|
||||
|
||||
|
||||
def test_a_stamped_distribution_defaults_off(tmp_path):
|
||||
_stamp(tmp_path)
|
||||
pol = policy.resolve(tmp_path)
|
||||
assert pol.enabled is False
|
||||
assert "distributed instance" in pol.reason
|
||||
|
||||
|
||||
def test_the_config_file_overrides_the_form_default(tmp_path):
|
||||
_stamp(tmp_path)
|
||||
_config_file(tmp_path, enabled=True)
|
||||
pol = policy.resolve(tmp_path)
|
||||
assert pol.enabled is True
|
||||
assert ".wikitool-telemetry.json" in pol.reason
|
||||
|
||||
|
||||
def test_the_env_var_overrides_the_config_file_in_both_directions(tmp_path, monkeypatch):
|
||||
_config_file(tmp_path, enabled=True)
|
||||
monkeypatch.setenv(policy.ENV_ENABLED, "0")
|
||||
assert policy.resolve(tmp_path).enabled is False
|
||||
|
||||
_stamp(tmp_path)
|
||||
_config_file(tmp_path, enabled=False)
|
||||
monkeypatch.setenv(policy.ENV_ENABLED, "1")
|
||||
assert policy.resolve(tmp_path).enabled is True
|
||||
|
||||
|
||||
def test_default_caps(tmp_path):
|
||||
pol = policy.resolve(tmp_path)
|
||||
assert pol.max_session_bytes == policy.DEFAULT_MAX_SESSION_BYTES
|
||||
assert pol.keep_sessions == policy.DEFAULT_KEEP_SESSIONS
|
||||
|
||||
|
||||
def test_the_config_file_can_set_both_caps(tmp_path):
|
||||
_config_file(tmp_path, max_session_bytes=1024, keep_sessions=5)
|
||||
pol = policy.resolve(tmp_path)
|
||||
assert pol.max_session_bytes == 1024
|
||||
assert pol.keep_sessions == 5
|
||||
|
||||
|
||||
def test_env_caps_override_the_config_file(tmp_path, monkeypatch):
|
||||
_config_file(tmp_path, max_session_bytes=1024, keep_sessions=5)
|
||||
monkeypatch.setenv(policy.ENV_MAX_SESSION_BYTES, "2048")
|
||||
monkeypatch.setenv(policy.ENV_KEEP_SESSIONS, "9")
|
||||
pol = policy.resolve(tmp_path)
|
||||
assert pol.max_session_bytes == 2048
|
||||
assert pol.keep_sessions == 9
|
||||
|
||||
|
||||
def test_a_non_positive_or_unparsable_override_is_ignored(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv(policy.ENV_MAX_SESSION_BYTES, "0")
|
||||
monkeypatch.setenv(policy.ENV_KEEP_SESSIONS, "not-a-number")
|
||||
pol = policy.resolve(tmp_path)
|
||||
assert pol.max_session_bytes == policy.DEFAULT_MAX_SESSION_BYTES
|
||||
assert pol.keep_sessions == policy.DEFAULT_KEEP_SESSIONS
|
||||
|
||||
|
||||
def test_a_malformed_config_file_is_treated_as_absent(tmp_path):
|
||||
(tmp_path / ".wikitool-telemetry.json").write_text("{not json", encoding="utf-8")
|
||||
pol = policy.resolve(tmp_path)
|
||||
assert pol.enabled is True # falls through to the dev-checkout form default
|
||||
|
||||
|
||||
def test_the_filesystem_half_is_cached_per_root(tmp_path):
|
||||
"""Writing the stamp *after* the first resolve must not retroactively
|
||||
change the cached answer - `reset_cache()` is required to see it."""
|
||||
first = policy.resolve(tmp_path)
|
||||
assert first.enabled is True
|
||||
_stamp(tmp_path)
|
||||
still_cached = policy.resolve(tmp_path)
|
||||
assert still_cached.enabled is True
|
||||
policy.reset_cache()
|
||||
assert policy.resolve(tmp_path).enabled is False
|
||||
|
||||
|
||||
def test_the_env_override_is_always_live_even_with_a_cached_root(tmp_path, monkeypatch):
|
||||
"""The env var must never need a `reset_cache()` to take effect - only the
|
||||
filesystem-derived half is cached."""
|
||||
assert policy.resolve(tmp_path).enabled is True
|
||||
monkeypatch.setenv(policy.ENV_ENABLED, "0")
|
||||
assert policy.resolve(tmp_path).enabled is False
|
||||
monkeypatch.delenv(policy.ENV_ENABLED)
|
||||
assert policy.resolve(tmp_path).enabled is True
|
||||
|
||||
|
||||
def test_resolve_defaults_to_config_root(tmp_path, monkeypatch):
|
||||
from chemenu import config
|
||||
|
||||
monkeypatch.setattr(config, "ROOT", tmp_path)
|
||||
_stamp(tmp_path)
|
||||
assert policy.resolve().enabled is False
|
||||
@@ -51,7 +51,8 @@ CHANGES_FILENAME = "CHANGES.md"
|
||||
# Written into an exported distribution by `dist export`, and committed with
|
||||
# it: an instance has to keep knowing which stack it came from, or update
|
||||
# detection has nothing to compare against. Machine-written, never hand-edited.
|
||||
RELEASE_STAMP_FILENAME = ".wikitool-release.json"
|
||||
# Defined in `config.py`, not here - see that constant's own comment.
|
||||
RELEASE_STAMP_FILENAME = config.RELEASE_STAMP_FILENAME
|
||||
STAMP_SCHEMA = 1
|
||||
|
||||
# Where `version check` looks when neither the environment nor a release stamp
|
||||
|
||||
Reference in New Issue
Block a user