diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 8ce9d34..e75f860 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -279,3 +279,18 @@ jobs: echo "reports/telemetry/ exists in a fresh distributed instance - telemetry should default off" exit 1 fi + + - name: The bug-report collector works in the distribution + # tools/bugreport.py is the one part of the stack that must run when + # nothing else does, so it is exercised as shipped: from the export, + # on the runner's plain python3, in the instance the step above set up. + # The unit tests cover what it collects; this covers that it arrives. + run: | + set -eu + cd "$DIST_DIR" + python3 tools/bugreport.py --no-trace + bundle=$(ls -d reports/bugreport-*/ | head -n 1) + test -f "$bundle/MANIFEST.md" + test -f "$bundle/environment.json" + grep -q '`wikitool` started' "$bundle/MANIFEST.md" + ls reports/bugreport-*.zip diff --git a/CHANGES.md b/CHANGES.md index 6ab7deb..e1f35ee 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -59,7 +59,7 @@ concern - readable here, never shipped as something to parse. --- -## 8.0.0-beta.4 - 2026-09-30 - Demo corpus follows the decided project pages: three states, seed with their items (#156) +## 8.0.0-beta.5 - 2026-09-30 - Bug-report collector: tools/bugreport.py and instructions/bug-report.md **Author:** Torben Nehmer @@ -80,6 +80,7 @@ concern - readable here, never shipped as something to parse. - Budget gate and loop-breaker refusals exit without a traceback - Super Productivity API path: unwrap the {ok, data} envelope, exclude the inbox project, ready-aware health (#162) - Live tracker suite: WIKITOOL_TASKS_CONFIG override, real-tracker tests for Super Productivity and CalDAV, nightly workflow and test image +- Bug-report collector: tools/bugreport.py and instructions/bug-report.md **Low impact** - version bump no longer points at version release in its output @@ -111,6 +112,34 @@ concern - readable here, never shipped as something to parse. - Demo corpus follows the decided project pages: three states, seed with their items (#156) +### Bug-report collector: tools/bugreport.py and instructions/bug-report.md (Gitea #157) + +A failure on one machine used to reach the maintainer as a description. `tools/bugreport.py` now +collects the first round of answers into one bundle, and it does so when `wikitool` itself does +not start: it uses the standard library only, imports nothing from `chemenu`, and keeps to +Python 3.8 syntax. + +- **Four layers.** The environment (OS, every Python and shell found, harness, `PATH`, git + configuration, venv, line endings, on Windows also long paths, execution policy and + mark-of-the-web), the stack (`VERSION`, the `.wikitool-*.json` files, git status and log, the + shape of `kb/` and `raw/` with path lengths and names that break on Windows), verbatim output of + `version show`, `doctor`, `budget status`, `instructions verify` and `docs verify` plus the + caller's session trace, and the agent's chronology with any transcripts. The result is + `reports/bugreport-/` and a zip beside it. +- **Secrets are always removed; page titles are kept out of everything the script generates** + unless `--titles` is given. Environment variable names are all recorded, values only for a + fixed list. The trace, the chronology and the transcripts are marked in the manifest as + possibly containing page content and titles. Nothing is uploaded. +- **Its own session id.** The two counting calls run under `WIKITOOL_SESSION_ID=bugreport-`, + so collecting a report neither spends nor is refused by the caller's budget. `gates.md` § + "Taking a new session id" names this as the second permitted case, for that script alone. +- **`instructions/bug-report.md`** (`manual: true`) is the agent's side: what to tell the user, + a fact-only chronology template, and the rule to stop before sending anything. `setup-instance.md` + and `upgrade-instance.md` offer it at a failure that has no obvious cause. +- `reports/CONTRACT.md` names the bundle as a third kind of output, `tools/README.md` explains why the + script sits beside the package, `INSTALL.md` has a troubleshooting entry, and CI runs the script + from the exported distribution. + ### Demo corpus follows the decided project pages: three states, seed with their items (Gitea #156) The first build of the live tracker suite shipped two demo project pages whose names came from a diff --git a/INSTALL.md b/INSTALL.md index 74de06e..841d198 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -473,6 +473,13 @@ tools/wikitool instructions verify als beim Mass-Update-Gate gibt es hier **keinen Token und keine Flagge** - stimmt das Ziel wirklich, trägt der Mensch dessen URL selbst in die Datei ein. Ein Agent, der die Datei anfasst, um an der Verweigerung vorbeizukommen, öffnet ein Gate aus eigenem Antrieb. +- **Setup oder Update schlägt fehl und ich will es melden** - den Agenten + [instructions/bug-report.md](instructions/bug-report.md) ausführen lassen, oder direkt + `python3 tools/bugreport.py`. Das Skript braucht nur Python 3.8 oder neuer, läuft auch ohne + funktionierendes `wikitool` und schreibt ein Bündel nach `reports/bugreport-/` + samt Zip. Geheimnisse werden entfernt, Seitentitel bleiben draußen (`--titles` nimmt sie mit); + das Bündel enthält trotzdem Maschinen-, Benutzer- und Pfadnamen. Es wird nirgends hochgeladen - + den Kanal wählst du selbst. - **Ich will am Tool-Stack selbst weiterarbeiten (nicht nur Wiki-Inhalt betreiben)** - eine neue Instanz hat dafür keinen Weg: `dist export` lässt `instructions/dev/` (Stack-Entwicklung, inkl. der vendorten `commonplace/`-Wissensbasis) bewusst und dauerhaft weg, ohne diff --git a/VERSION b/VERSION index 4e54dd7..7a7a031 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -8.0.0-beta.4 +8.0.0-beta.5 diff --git a/instructions/bug-report.md b/instructions/bug-report.md new file mode 100644 index 0000000..f36f1b1 --- /dev/null +++ b/instructions/bug-report.md @@ -0,0 +1,135 @@ +--- +type: types/instruction.md +name: bug-report +description: How to collect a bug-report bundle when the stack misbehaves on this machine - run tools/bugreport.py, write a fact-only chronology, tell the user what the bundle contains, and stop short of sending it anywhere. +manual: true +--- +# Collect a bug report + +When setup, an upgrade or a command fails on one machine and works on another, the person who has to +fix it sees nothing of what happened here. This procedure produces one bundle that answers the first +round of their questions - which machine, which Python, which shell, which harness, what the stack +looked like, what `wikitool` printed - so that the report is not a guessing game. + +**Run this only when asked, by name, or when the user agrees to it after a failure.** It is +`manual: true` on purpose: the bundle contains private data, and whether to produce one is the +user's decision, not the agent's. Nothing links to this file from `AGENTS.md` or a skill, apart from +the pointers at the failure decision points of [setup-instance.md](setup-instance.md) and +[upgrade-instance.md](upgrade-instance.md), which only offer it. + + +## Contents + +- [What the bundle holds](#what-the-bundle-holds) +- [When to run](#when-to-run) +- [Steps](#steps) +- [Chronology template](#chronology-template) +- [Decision points](#decision-points) +- [Scope](#scope) + + +## What the bundle holds + +`tools/bugreport.py` is a standalone script. It uses the standard library only and imports nothing +from the stack, so it still runs when `wikitool` does not start - no venv, a broken package, a Python +that is too old for the stack. It writes `reports/bugreport-/` and a zip beside it, in +four layers: + +| Layer | Files | Holds | +|-------|-------|-------| +| 1 Environment | `environment.json` | OS, every Python and shell found, harness, `PATH`, environment variable names (values only for a fixed list), git configuration, venv, line endings, on Windows also long paths, execution policy and mark-of-the-web | +| 2 Stack | `stack.json`, `tree-structure.json` | `VERSION`, the `.wikitool-*.json` files with secrets removed, git status and the last commits, and the shape of `kb/` and `raw/` (counts, depths, path lengths, names that break on Windows) | +| 3 wikitool | `wikitool/*.txt`, `trace.jsonl` | Verbatim output of `version show`, `doctor`, `budget status`, `instructions verify` and `docs verify`, and the caller's session trace. If `version show` fails, `wikitool` counts as not started and nothing else runs | +| 4 Chronology | `CHRONOLOGY.md`, `transcripts/` | What the agent did and saw, and harness transcripts if the user asked for them | + +`MANIFEST.md` lists every file and marks the ones that may contain page content and titles: the trace, +the chronology and the transcripts. + +Two rules hold for everything the script generates itself. **Secrets are always removed**: values of +keys that look like a token, password, secret, key or auth entry, credentials in URLs, and every value +of that kind found while collecting is also replaced wherever else it turns up. **Page titles are kept +out** unless `--titles` is given: paths under `kb/` and `raw/` are replaced by their shape +(depth, length, whether they hold a space or a non-ASCII character) and `[[wikilinks]]` by the same +flags. A title that stands as bare prose is not found - which is why the trace, the chronology and the +transcripts are marked instead. + +## When to run + +- Setup or an upgrade failed and the user wants to report it. +- A command fails in a way that looks tied to the machine, and the user asks for a report. +- The user asks for a bug report by name. + +## Steps + +1. **Tell the user what is about to happen**, in the instance's KB language: a bundle will be written + under `reports/`, it is not pseudonymised, it holds machine, user and path names and the git + remotes, and it is not sent anywhere. Ask whether the session trace, page titles and transcripts + may go in. The defaults are: trace in, titles out, no transcripts. + +2. **Write the chronology** to a file under `reports/` (which is gitignored) from the + [template](#chronology-template) below. Facts only - no diagnosis. If the session's own history is + too long to reconstruct, say what is missing instead of filling the gap. + +3. **Run the collector**, with whichever Python 3.8 or later the machine has: + + ```bash + python3 tools/bugreport.py --chronology reports/chronology.md + ``` + + Add `--no-trace` if the user declined the trace, `--titles` if titles may stay, + `--transcript ` (repeatable) for a transcript the user pointed at, `--session ` to take a + trace other than the caller's. `python3` may be `python` or `py -3` on Windows. If no Python starts + at all, that is the report: give the user the exact error text, verbatim. + +4. **Do not imitate the collector's session id.** It runs its counting `wikitool` calls under + `WIKITOOL_SESSION_ID=bugreport-` itself; do not export that variable, or any `bugreport-*` + one, in the session. The exception it gets in [gates.md](gates.md) § "Taking a new session id" + belongs to the script alone. + +5. **Report the result.** Quote the bundle path, the archive path and the privacy notice the script + prints, name the gaps in `MANIFEST.md`, and tell the user to read the bundle before sharing it. + Then stop: the channel - a tracker issue, an email, a chat - is the user's choice, and the agent + never uploads the bundle. + +## Chronology template + +```markdown +# Chronology + +## Goal +One sentence: what the user asked for, e.g. "Set up a new instance from a fresh clone." + +## Environment as the agent saw it +Harness, shell, OS, anything the user said about the machine that the collector cannot know. + +## Steps +1. `` - exit code, and the first line of the error or output that mattered, verbatim. +2. ... + +## Expected and observed +- Expected: what the instruction said would happen. +- Observed: what happened instead, verbatim where it is short. + +## Changes made by hand +Every file edited or created outside a `wikitool` command, and every setting changed, in order. + +## Not known +What the agent could not find out, or did not check. +``` + +Facts only: no page content, no guessed cause, no advice. + +## Decision points + +- **The user declines the trace?** Run with `--no-trace`. The manifest records the exclusion. +- **The user wants the report to name pages?** Run with `--titles`; the bundle then also carries + `tree-paths.txt`, which lists every path under `kb/` and `raw/`. +- **The collector exits 1?** It could not write the bundle (a missing input file, a full disk). Read + the message, fix the cause, retry once, then report the exact error. +- **A `wikitool` output in the bundle looks wrong or refuses to run?** Do not re-run it to see more. + The bundle records what happened; that is the report. + +## Scope + +Covers producing the bundle. It does not cover reading a bundle someone else sent, triaging the +report, or filing it - all of that is the maintainer's side and the user's choice of channel. diff --git a/instructions/gates.md b/instructions/gates.md index 60a0219..a6a2bb9 100644 --- a/instructions/gates.md +++ b/instructions/gates.md @@ -215,6 +215,14 @@ never in response to a gate refusal.** The plan is the human approval the gate w have to ask for; a refusal means that approval has not been given yet. If you are tempted to re-export the variable after an `ERROR` line, that is the gate working. +**One tool takes a new id itself, for a fixed set of read commands.** `tools/bugreport.py` +([bug-report.md](bug-report.md)) runs `instructions verify` and `docs verify` under +`WIKITOOL_SESSION_ID=bugreport-`, so that collecting a report neither spends the +caller's budget nor is refused by it at exactly the moment something has gone wrong. That is +permitted because the script sets the id for those two commands only, never in the caller's +shell, and the set is a constant in the script. It is not a precedent for an agent: an agent that +exports a `bugreport-*` id, or any other, outside the two cases above is opening the gate. + Background: the [[Iteration and Cost Limits]] concept page in `kb/`. ## Scope diff --git a/instructions/setup-instance.md b/instructions/setup-instance.md index 37283ac..e1e54b4 100644 --- a/instructions/setup-instance.md +++ b/instructions/setup-instance.md @@ -305,6 +305,10 @@ and ready for its first ingest. afterwards are `wiki-ingest`, `wiki-query`, `wiki-manage`, `wiki-lint`, `wiki-status` and `gtd-weekly-review` available. +**A step fails and the cause is not obvious?** Do not improvise around it (invariant 7). Offer the +user a bug report - [bug-report.md](bug-report.md) - and run it only if they agree; the collector +works even when `wikitool` does not start. + ## Scope Applies only to an empty distribution produced by `dist export`. For an existing clone of this diff --git a/instructions/upgrade-instance.md b/instructions/upgrade-instance.md index 65a0d15..36df18a 100644 --- a/instructions/upgrade-instance.md +++ b/instructions/upgrade-instance.md @@ -247,6 +247,9 @@ fresh clone ([bootstrap.md](bootstrap.md)), and not for the clone-with-upstream intermediate state is legitimate and `migrate status` resumes it; what is not legitimate is recording a migration with `migrate done` that was not carried out - the version then describes a shape the corpus is not in. +- **A step fails and the cause is not obvious?** Stop rather than improvise, and offer the user a + bug report - [bug-report.md](bug-report.md). It runs even when `wikitool` does not start, and + only when the user agrees. - **`doctor` reports `kb-version` behind `VERSION` after everything is done?** Correct when the release's chain was empty or carried only `offered` entries: an offer changes a file the instance owns, not the shape of its content, so the content version stays where it was. diff --git a/reports/CONTRACT.md b/reports/CONTRACT.md index f57e61b..1a4e4c7 100644 --- a/reports/CONTRACT.md +++ b/reports/CONTRACT.md @@ -1,12 +1,14 @@ # reports/ - Generated Output -Derived output that must stay out of git. Two kinds live here: +Derived output that must stay out of git. Three kinds live here: - **Lint reports**, written by `tools/wikitool lint --markdown "reports/Lint Report .md"`. - **Traces**, under `reports/telemetry//trace.jsonl` - the append-only record of what a session did, written by `wikitool` itself and by the harness hooks. See [../EVALS.md](../EVALS.md) for the event contract and what is redacted. +- **Bug-report bundles**, under `reports/bugreport-/` and a zip beside each, written by + `tools/bugreport.py` on request - see [../instructions/bug-report.md](../instructions/bug-report.md). **Everything in this directory except this file is gitignored.** A lint report is a derived copy of recomputable truth: its structural sections can be regenerated from the tree at any @@ -39,12 +41,27 @@ installation-form default (on for a dev checkout, off for a distributed instance keeps lengths and digests instead of text. See [../EVALS.md](../EVALS.md) § "Whether it runs at all" for the full precedence and both quantity caps below. +## Bug-report bundles + +A bundle is a snapshot of one machine at one moment, made so that someone else can read what +happened here. It is not recomputable and not durable, and it **contains private data**: machine, +user and path names, `PATH` entries, git remotes and commit subjects, and - when included - the +session trace, the chronology and transcripts, which may hold page content and titles. Secrets are +removed by the collector; the rest is the reader's to check before a bundle leaves the machine. +Nothing uploads it: the channel is the user's choice. + +The collector's own `wikitool` calls run under the session id `bugreport-`, so a +`reports/telemetry/bugreport-*` directory is that run's trace and belongs to no session of yours. + ## Retention **Lint reports: none.** Old ones are local scratch; delete them freely. There is nothing to retire with `wikitool rm`, because no report is ever a wiki page - `lint-report` is a contract-only type-spec with no `base_dir:` and cannot be instantiated under `kb/`. +**Bug-report bundles: none.** Delete them freely once they have been read or sent; nothing refers +to one afterwards. + **Traces: two enforced caps, applied by the writer itself, never by a separate cleanup pass.** A byte cap per session trace (default 5 MiB, `WIKI_TRACE_MAX_SESSION_BYTES`) and a retention limit on the number of session directories under `reports/telemetry/` (default 250, diff --git a/tools/README.md b/tools/README.md index 886d01d..ee1b678 100644 --- a/tools/README.md +++ b/tools/README.md @@ -60,6 +60,13 @@ tools/ tests/ pytest suite ``` +**A script beside the package.** `bugreport.py` is not part of `chemenu` and imports nothing +from it: it is the bug-report collector ([../instructions/bug-report.md](../instructions/bug-report.md)), +and the case it exists for is a checkout where `wikitool` does not start. It therefore uses the +standard library only, keeps to Python 3.8 syntax so that an old interpreter can still run it, and +exits 0 or 1 - never 42, since it opens no gate. `dist export` ships it with the rest of `tools/`; +its tests (`tests/test_bugreport.py`) run it on a bare interpreter (`-I -S`) to keep that promise. + **Two consumers, one core.** The CLI is not the only caller any more. The cores (`search/service.py`, `lint_core.py`, `types_core.py`, `catalog.py`) hold what decides an answer and import no `typer` and no `rich`; the modules under `commands/` turn diff --git a/tools/bugreport.py b/tools/bugreport.py new file mode 100644 index 0000000..abfde20 --- /dev/null +++ b/tools/bugreport.py @@ -0,0 +1,942 @@ +#!/usr/bin/env python3 +"""Collect a bug-report bundle from this checkout. + +Runs on the base Python with the standard library only, and imports nothing from +`chemenu`: the case it exists for is a checkout where `wikitool` does not start +- no venv, a broken package, a Python that is too old. Syntax stays at Python +3.8 so that even an old interpreter can still produce a report. + + python tools/bugreport.py [--chronology FILE] [--transcript FILE]... + [--session ID | --no-trace] [--titles] + [--root DIR] [--out DIR] + +The bundle is `/bugreport-/` plus a zip beside it, in four +layers: the environment, the stack, what `wikitool` prints (if it starts), and +the agent's chronology. Secrets are always removed. Page titles are kept out of +everything this script generates unless `--titles` is given. Nothing is uploaded. + +Exit 0 = bundle written, 1 = it could not be written. No exit 42: this script +opens no gate. See instructions/bug-report.md. +""" +from __future__ import annotations + +import argparse +import datetime +import json +import os +import platform +import re +import shutil +import subprocess +import sys +import unicodedata +import zipfile +from pathlib import Path + +TIMEOUT_SECONDS = 300 +PROBE_TIMEOUT_SECONDS = 20 +REMOVED = "" +NOT_COLLECTED = "" +SESSION_ENV = "WIKITOOL_SESSION_ID" +HARNESS_SESSION_ENV = "CLAUDE_CODE_SESSION_ID" + +SECRET_NAME = re.compile(r"token|password|secret|key|auth", re.IGNORECASE) + +HARNESS_MARKERS = ( + (HARNESS_SESSION_ENV, "claude-code"), +) + +ENV_ALLOW_EXACT = frozenset( + { + "PATH", "PATHEXT", "SHELL", "TERM", "TERM_PROGRAM", "TERM_PROGRAM_VERSION", + "COLORTERM", "LANG", "LANGUAGE", "VIRTUAL_ENV", "MSYSTEM", "COMSPEC", + "PSMODULEPATH", + } + | {name for name, _ in HARNESS_MARKERS} +) +ENV_ALLOW_PREFIX = ("LC_", "PYTHON", "WIKI_", "WIKITOOL_", "CHEMENU_") + +# Names under kb/ that the stack generates or owns: not titles, and unreadable +# in a bundle if they are masked. +STACK_NAMES = frozenset( + { + "CONTRACT.md", "CONVENTIONS.md", "CONVENTIONS.md.template", "COLLECTION.md", + "COLLECTION.md.template", "INDEX.md", "index.md", "log.md", "provenance.md", + ".gitkeep", + } +) +CONTENT_DIRS = ("kb", "raw") +CONTENT_COMMIT_DIRS = ("kb", "raw", "work") + +MAY_CONTAIN_CONTENT = "may contain page content and titles" + +PRIVACY_NOTICE = ( + "This bundle is not pseudonymised. It contains private data: machine, user and path " + "names, PATH entries, git remotes and commit subjects and - if included - the session " + "trace, the chronology and transcripts, which may contain page content and titles. " + "Secrets are removed, but read the bundle before you share it. Choose the channel " + "yourself; neither this tool nor the agent uploads anything." +) + +WINDOWS_RESERVED = re.compile(r"^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\..*)?$", re.IGNORECASE) +WINDOWS_FORBIDDEN = re.compile(r'[<>:"|?*\x00-\x1f]') +WIKILINK = re.compile(r"\[\[([^\]\n]+)\]\]") +URL_USERINFO = re.compile(r"(?P[A-Za-z][A-Za-z0-9+.\-]*)://(?P[^/\s@\"'<>]+)@") +AUTH_HEADER = re.compile(r"(?im)(authorization\s*[:=]\s*)(?:(?:bearer|basic|token)\s+)?\S+") +BEARER = re.compile(r"(?i)\b(bearer)\s+[A-Za-z0-9._~+/=\-]{8,}") + + +# --------------------------------------------------------------------------- secrets + + +class Redactor: + """Every secret value found while collecting, replaced again in a final pass.""" + + def __init__(self) -> None: + self.values = set() + + def add(self, value) -> None: + if isinstance(value, str) and len(value) >= 4: + self.values.add(value) + + def add_any(self, node) -> None: + if isinstance(node, dict): + for item in node.values(): + self.add_any(item) + elif isinstance(node, (list, tuple)): + for item in node: + self.add_any(item) + else: + self.add(node) + + def scrub_urls(self, text: str) -> str: + def repl(match): + scheme, userinfo = match.group("scheme"), match.group("userinfo") + if scheme.lower() in ("http", "https"): + self.add(userinfo) + for part in userinfo.split(":"): + self.add(part) + return scheme + "://" + if ":" in userinfo: + user, password = userinfo.split(":", 1) + self.add(password) + return scheme + "://" + user + "@" + return match.group(0) + + return URL_USERINFO.sub(repl, text) + + def scrub(self, text: str) -> str: + text = self.scrub_urls(text) + text = AUTH_HEADER.sub(lambda m: m.group(1) + REMOVED, text) + text = BEARER.sub(lambda m: m.group(1) + " " + REMOVED, text) + forms = set() + for value in self.values: + forms.add(value) + escaped = json.dumps(value, ensure_ascii=False)[1:-1] + if escaped != value: + forms.add(escaped) + for form in sorted(forms, key=len, reverse=True): + text = text.replace(form, REMOVED) + return text + + def clean_json(self, node): + if isinstance(node, dict): + out = {} + for key, value in node.items(): + if SECRET_NAME.search(str(key)): + self.add_any(value) + out[key] = REMOVED + else: + out[key] = self.clean_json(value) + return out + if isinstance(node, list): + return [self.clean_json(item) for item in node] + if isinstance(node, str): + return self.scrub_urls(node) + return node + + +# ------------------------------------------------------------------------ title shield + + +def _shape(rel: str) -> str: + parts = rel.split("/") + name = parts[-1] + stem, dot, suffix = name.rpartition(".") + flags = ["depth=%d" % len(parts), "len=%d" % len(rel)] + if " " in rel: + flags.append("space") + if any(ord(ch) > 127 for ch in rel): + flags.append("non-ascii") + return "%s/<%s>%s" % (parts[0], " ".join(flags), (dot + suffix) if dot and stem else "") + + +class TitleShield: + """Replaces the relative path of a file under kb/ or raw/ with its shape. + + A title that stands as bare prose is not found, on purpose: replacing titles + word by word would hit a title like "Git" everywhere in the bundle. + """ + + def __init__(self, rel_paths) -> None: + self.forms = {} + for rel in rel_paths: + shape = _shape(rel) + for form in (rel, rel.replace("/", "\\"), rel.replace("/", "\\\\")): + self.forms[form] = shape + self.lengths = sorted({len(form) for form in self.forms}, reverse=True) + self.start = re.compile(r"(?:%s)(?:/|\\)" % "|".join(CONTENT_DIRS)) + + def apply(self, text: str) -> str: + if self.forms: + out, last = [], 0 + for match in self.start.finditer(text): + i = match.start() + if i < last: + continue + for length in self.lengths: + shape = self.forms.get(text[i:i + length]) + if shape: + out.append(text[last:i]) + out.append(shape) + last = i + length + break + out.append(text[last:]) + text = "".join(out) + return WIKILINK.sub(lambda m: "[[]]" % _title_flags(m.group(1)), text) + + +def _title_flags(title: str) -> str: + flags = ["len=%d" % len(title)] + if " " in title: + flags.append("space") + if any(ord(ch) > 127 for ch in title): + flags.append("non-ascii") + return " ".join(flags) + + +# ---------------------------------------------------------------------------- helpers + + +class Result: + def __init__(self, returncode, stdout, stderr, note=None) -> None: + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + self.note = note + + @property + def ok(self) -> bool: + return self.returncode == 0 + + def first_line(self) -> str: + for text in (self.stderr, self.stdout, self.note or ""): + for line in text.splitlines(): + if line.strip(): + return line.strip() + return "" + + +def _decode(data) -> str: + if data is None: + return "" + if isinstance(data, str): + return data + return data.decode("utf-8", errors="replace") + + +def run(cmd, cwd, env=None, timeout=PROBE_TIMEOUT_SECONDS) -> Result: + try: + proc = subprocess.run( + cmd, cwd=str(cwd), env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout, + ) + except subprocess.TimeoutExpired as exc: + return Result(None, _decode(exc.stdout), _decode(exc.stderr), + "timed out after %d s" % timeout) + except OSError as exc: + return Result(None, "", "", "could not start: %s" % exc) + return Result(proc.returncode, _decode(proc.stdout), _decode(proc.stderr)) + + +def read_text(path: Path) -> str: + with open(str(path), "r", encoding="utf-8", errors="replace", newline="") as handle: + return handle.read() + + +def write_text(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(str(path), "w", encoding="utf-8", newline="\n") as handle: + handle.write(text) + + +def dump_json(data) -> str: + return json.dumps(data, indent=2, ensure_ascii=False) + "\n" + + +def guarded(fn, *args): + """A broken layer must not take the whole report down.""" + try: + return fn(*args) + except Exception as exc: # noqa: BLE001 - the report is the point + return {"collector_error": "%s: %s" % (type(exc).__name__, exc)} + + +def session_slug(value: str) -> str: + return re.sub(r"[^A-Za-z0-9._-]+", "__", value).strip("_") or "unknown" + + +# ------------------------------------------------------------------- layer 1: environment + + +def _version_line(cmd, cwd) -> str: + result = run(cmd, cwd) + return result.first_line() if result.ok or result.first_line() else (result.note or "") + + +def collect_environment(root: Path, red: Redactor) -> dict: + data = { + "os": { + "platform": platform.platform(), + "system": platform.system(), + "release": platform.release(), + "version": platform.version(), + "machine": platform.machine(), + }, + "python": _collect_python(root), + "shells": _collect_shells(root), + "harness": _collect_harness(), + "path_entries": [ + {"entry": entry, "exists": os.path.isdir(entry)} + for entry in os.environ.get("PATH", "").split(os.pathsep) if entry + ], + "env": _collect_env(red), + "tools_config": _collect_tools_config(root, red), + "git": _collect_git_config(root, red), + "venv": _collect_venv(root, red), + "line_endings": _collect_line_endings(root), + "gitattributes": _read_optional(root / ".gitattributes", red), + "windows": _collect_windows(root), + } + return data + + +def _collect_python(root: Path) -> dict: + running = { + "executable": sys.executable, + "version": sys.version.replace("\n", " "), + "version_info": list(sys.version_info[:3]), + } + candidates = [] + for name, extra in (("python3", []), ("python", []), ("py", ["-3"])): + path = shutil.which(name) + entry = {"name": " ".join([name] + extra), "path": path} + if path: + entry["version"] = _version_line([path] + extra + ["--version"], root) + entry["windows_store_alias"] = "windowsapps" in path.lower() + candidates.append(entry) + return {"running": running, "candidates": candidates} + + +def _collect_shells(root: Path) -> dict: + shells = {"SHELL": os.environ.get("SHELL"), "COMSPEC": os.environ.get("COMSPEC")} + for name in ("bash", "zsh", "fish"): + path = shutil.which(name) + shells[name] = {"path": path, + "version": _version_line([path, "--version"], root) if path else None} + for name in ("pwsh", "powershell"): + path = shutil.which(name) + shells[name] = { + "path": path, + "version": _version_line( + [path, "-NoProfile", "-Command", "$PSVersionTable.PSVersion.ToString()"], root + ) if path else None, + } + return shells + + +def _collect_harness() -> dict: + detected = [ + {"marker": name, "harness": harness} + for name, harness in HARNESS_MARKERS if os.environ.get(name) + ] + if os.environ.get("TERM_PROGRAM") == "vscode": + detected.append({"marker": "TERM_PROGRAM=vscode", "harness": "vscode terminal"}) + return {"detected": detected or "unrecognised"} + + +def _env_allowed(name: str) -> bool: + upper = name.upper() + return upper in ENV_ALLOW_EXACT or upper.startswith(ENV_ALLOW_PREFIX) + + +def _collect_env(red: Redactor) -> dict: + env = {} + for name in sorted(os.environ): + value = os.environ[name] + if SECRET_NAME.search(name): + red.add(value) + env[name] = REMOVED if _env_allowed(name) else NOT_COLLECTED + elif _env_allowed(name): + env[name] = value + else: + env[name] = NOT_COLLECTED + return env + + +def _collect_tools_config(root: Path, red: Redactor) -> dict: + path = root / ".wikitool-tools.json" + if not path.is_file(): + return {"status": "absent"} + try: + parsed = json.loads(read_text(path)) + except ValueError as exc: + return {"status": "unparseable", "error": str(exc)} + checks = [] + + def walk(node, where): + if isinstance(node, dict): + for key, value in node.items(): + walk(value, where + [str(key)]) + elif isinstance(node, list): + for index, value in enumerate(node): + walk(value, where + [str(index)]) + elif isinstance(node, str) and os.path.isabs(node): + checks.append({"at": ".".join(where), "path": node, "exists": os.path.exists(node)}) + + walk(parsed, []) + return {"status": "present", "content": red.clean_json(parsed), "path_checks": checks} + + +def _collect_git_config(root: Path, red: Redactor) -> dict: + version = run(["git", "--version"], root) + if not version.ok: + return {"available": False, "error": version.note or version.first_line()} + config = {} + for key in ("core.autocrlf", "core.longpaths", "core.filemode"): + result = run(["git", "config", "--get", key], root) + config[key] = result.stdout.strip() if result.ok else "(unset)" + name = run(["git", "config", "--get", "user.name"], root) + config["user.name is set"] = bool(name.ok and name.stdout.strip()) + remotes = run(["git", "remote", "-v"], root) + return { + "available": True, + "version": version.stdout.strip(), + "config": config, + "remotes": red.scrub_urls(remotes.stdout).splitlines() if remotes.ok + else ["(not a repository, or git failed: %s)" % remotes.first_line()], + } + + +def _collect_venv(root: Path, red: Redactor) -> dict: + venv = root / "tools" / ".venv" + if not venv.is_dir(): + return {"present": False} + layout = ("bin" if (venv / "bin").is_dir() else None) or ( + "Scripts" if (venv / "Scripts").is_dir() else None) + cfg = venv / "pyvenv.cfg" + return { + "present": True, + "layout": layout or "unknown", + "python_found": any( + (venv / sub / exe).exists() + for sub, exe in (("bin", "python"), ("Scripts", "python.exe")) + ), + "pyvenv.cfg": read_text(cfg) if cfg.is_file() else None, + } + + +def _collect_line_endings(root: Path) -> dict: + tools = root / "tools" + files = [tools / "wikitool"] + (sorted(tools.glob("*.ps1")) if tools.is_dir() else []) + result = {} + for path in files: + if not path.is_file(): + continue + raw = path.read_bytes() + crlf = raw.count(b"\r\n") + result[path.name] = {"crlf": crlf, "lf_only": raw.count(b"\n") - crlf} + return result + + +def _read_optional(path: Path, red: Redactor): + return red.scrub_urls(read_text(path)) if path.is_file() else None + + +def _collect_windows(root: Path) -> dict: + if sys.platform != "win32": + return {"applicable": False} + data = {"applicable": True} + try: + import winreg # noqa: PLC0415 - only exists on Windows + + with winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, r"SYSTEM\CurrentControlSet\Control\FileSystem" + ) as key: + data["LongPathsEnabled"] = winreg.QueryValueEx(key, "LongPathsEnabled")[0] + except Exception as exc: # noqa: BLE001 + data["LongPathsEnabled"] = "unreadable: %s" % exc + pwsh = shutil.which("pwsh") + if pwsh: + policy = run([pwsh, "-NoProfile", "-Command", "Get-ExecutionPolicy -List | Out-String"], root) + data["execution_policy"] = policy.stdout.strip() or policy.first_line() + else: + data["execution_policy"] = "pwsh not found" + data["powershell_5_present"] = bool(shutil.which("powershell")) + marked = [] + tools = root / "tools" + if tools.is_dir(): + for path in sorted(p for p in tools.iterdir() if p.is_file()): + try: + with open(str(path) + ":Zone.Identifier", "r", encoding="utf-8", + errors="replace") as stream: + marked.append({"file": path.name, "zone": stream.read().strip()}) + except OSError: + pass + data["mark_of_the_web"] = marked + return data + + +# ------------------------------------------------------------------------ layer 2: stack + + +def scan_tree(root: Path, name: str): + """Structure of kb/ or raw/, and the relative paths it holds. No content.""" + top = root / name + if not top.is_dir(): + return {"present": False}, [] + files, dirs = [], 0 + depth_histogram, longest_rel, longest_abs = {}, 0, 0 + counts = { + "abs_path_over_240": 0, "abs_path_over_260": 0, "names_with_space": 0, + "names_non_ascii": 0, "names_windows_forbidden_chars": 0, + "names_trailing_dot_or_space": 0, "names_reserved_device": 0, + "names_not_nfc": 0, "case_collisions": 0, + } + + def judge(entry: str) -> None: + if " " in entry: + counts["names_with_space"] += 1 + if any(ord(ch) > 127 for ch in entry): + counts["names_non_ascii"] += 1 + if WINDOWS_FORBIDDEN.search(entry): + counts["names_windows_forbidden_chars"] += 1 + if entry.endswith((" ", ".")): + counts["names_trailing_dot_or_space"] += 1 + if WINDOWS_RESERVED.match(entry): + counts["names_reserved_device"] += 1 + if unicodedata.normalize("NFC", entry) != entry: + counts["names_not_nfc"] += 1 + + for current, subdirs, names in os.walk(str(top)): + entries = sorted(subdirs) + sorted(names) + seen = {} + for entry in entries: + seen.setdefault(entry.casefold(), []).append(entry) + judge(entry) + counts["case_collisions"] += sum(1 for group in seen.values() if len(group) > 1) + dirs += len(subdirs) + for entry in names: + absolute = os.path.join(current, entry) + rel = os.path.relpath(absolute, str(root)).replace(os.sep, "/") + files.append(rel) + depth = rel.count("/") + 1 + depth_histogram[str(depth)] = depth_histogram.get(str(depth), 0) + 1 + longest_rel = max(longest_rel, len(rel)) + longest_abs = max(longest_abs, len(os.path.abspath(absolute))) + if len(os.path.abspath(absolute)) > 240: + counts["abs_path_over_240"] += 1 + if len(os.path.abspath(absolute)) > 260: + counts["abs_path_over_260"] += 1 + summary = { + "present": True, "files": len(files), "directories": dirs, + "depth_histogram": depth_histogram, "longest_relative_path": longest_rel, + "longest_absolute_path": longest_abs, + } + summary.update(counts) + return summary, files + + +def collect_stack(root: Path, red: Redactor) -> dict: + version = root / "VERSION" + configs = {} + for path in sorted(root.glob(".wikitool-*.json")): + if path.name == ".wikitool-tools.json": + continue + try: + configs[path.name] = red.clean_json(json.loads(read_text(path))) + except ValueError: + configs[path.name] = {"unparseable": True, "bytes": path.stat().st_size} + return { + "VERSION": read_text(version).strip() if version.is_file() else "absent", + "config_files": configs, + "git": _collect_git_state(root), + } + + +def _collect_git_state(root: Path) -> dict: + quiet = ["git", "-c", "core.quotepath=off"] + inside = run(quiet + ["rev-parse", "--is-inside-work-tree"], root) + if not inside.ok: + return {"repository": False, "error": inside.first_line()} + branch = run(quiet + ["rev-parse", "--abbrev-ref", "HEAD"], root) + status = run(quiet + ["status", "--porcelain", "-z"], root) + log = run(quiet + ["log", "-n", "20", "--name-only", "--format=%x1e%h%x1f%s"], root) + return { + "repository": True, + "branch": branch.stdout.strip() if branch.ok else branch.first_line(), + "status": _parse_status(status.stdout) if status.ok else [status.first_line()], + "log": _parse_log(log.stdout) if log.ok else [log.first_line()], + } + + +def _parse_status(raw: str) -> list: + entries = raw.split("\0") + lines, i = [], 0 + while i < len(entries): + entry = entries[i] + i += 1 + if len(entry) < 4: + continue + code, path = entry[:2], entry[3:] + if code[0] in "RC" and i < len(entries): + lines.append("%s %s (from %s)" % (code, path, entries[i])) + i += 1 + else: + lines.append("%s %s" % (code, path)) + return lines + + +def _parse_log(raw: str) -> list: + lines = [] + for block in raw.split("\x1e"): + if not block.strip(): + continue + head, _, rest = block.partition("\n") + sha, _, subject = head.partition("\x1f") + files = [line for line in rest.splitlines() if line.strip()] + if files and all(f.startswith(tuple(d + "/" for d in CONTENT_COMMIT_DIRS)) for f in files): + parts = [] + for directory in CONTENT_COMMIT_DIRS: + count = sum(1 for f in files if f.startswith(directory + "/")) + if count: + parts.append("%s %d" % (directory, count)) + subject = "<content commit: %s files>" % ", ".join(parts) + lines.append("%s %s" % (sha.strip(), subject.strip())) + return lines + + +# ---------------------------------------------------------------------- layer 3: wikitool + + +def launcher_command(root: Path): + tools = root / "tools" + if sys.platform == "win32": + ps1, sh = tools / "wikitool.ps1", tools / "wikitool" + pwsh = shutil.which("pwsh") + if ps1.is_file() and pwsh: + return [pwsh, "-NoProfile", "-File", str(ps1)] + bash = shutil.which("bash") + if sh.is_file() and bash: + return [bash, str(sh)] + return None + launcher = tools / "wikitool" + return [str(launcher)] if launcher.is_file() else None + + +def resolve_trace_session(explicit): + if explicit: + return explicit, "--session" + for name in (SESSION_ENV, HARNESS_SESSION_ENV): + if os.environ.get(name): + return os.environ[name], name + return None, None + + +def trace_root(root: Path) -> Path: + override = os.environ.get("WIKI_TRACE_DIR") + return Path(override) if override else root / "reports" / "telemetry" + + +def collect_trace(root: Path, explicit): + """The caller's session trace, or a listing of what exists instead.""" + session, source = resolve_trace_session(explicit) + base = trace_root(root) + if session: + path = base / session_slug(session) / "trace.jsonl" + if path.is_file(): + return {"found": True, "session": session, "source": source, + "text": read_text(path)} + listing = [] + if base.is_dir(): + for child in sorted(base.iterdir()): + if child.is_dir() and not child.name.startswith("bugreport-"): + trace = child / "trace.jsonl" + listing.append({"session_directory": child.name, + "bytes": trace.stat().st_size if trace.is_file() else 0}) + return {"found": False, "session": session, "source": source, "listing": listing} + + +WIKITOOL_CALLS = ( + ("version show", ("version", "show"), "inherited"), + ("doctor", ("doctor",), "inherited"), + ("budget status", ("budget", "status"), "inherited"), + ("instructions verify", ("instructions", "verify"), "own"), + ("docs verify", ("docs", "verify"), "own"), +) + + +def run_wikitool(root: Path, launcher, own_session: str) -> dict: + """Five fixed read commands. `version show` is the startup probe: if it fails, + `wikitool` counts as not started and nothing else is run.""" + outputs = [] + started = False + reason = None + for label, args, session in WIKITOOL_CALLS: + env = dict(os.environ) + if session == "own": + env[SESSION_ENV] = own_session + result = run(launcher + list(args), root, env=env, timeout=TIMEOUT_SECONDS) + prefix = "%s=%s " % (SESSION_ENV, own_session) if session == "own" else "" + outputs.append({ + "label": label, + "file": "wikitool/%s.txt" % label.replace(" ", "-"), + "session": ("%s (set by the collector)" % own_session) if session == "own" + else "inherited from the caller", + "text": ( + "command: %s%s\nsession: %s\nexit: %s\n%s\n--- stdout ---\n%s\n--- stderr ---\n%s\n" + % (prefix, " ".join(launcher + list(args)), + "own" if session == "own" else "inherited", + result.returncode if result.returncode is not None else result.note, + ("note: " + result.note) if result.note and result.returncode is not None + else "", result.stdout, result.stderr) + ), + }) + if label == "version show": + if not result.ok: + reason = "version show %s: %s" % ( + ("exit %s" % result.returncode) if result.returncode is not None + else result.note, result.first_line()) + break + started = True + return {"started": started, "reason": reason, "outputs": outputs} + + +# ------------------------------------------------------------------------------ bundle + + +class Bundle: + def __init__(self, directory: Path) -> None: + self.directory = directory + self.files = [] + + def add(self, rel: str, text: str, layer: str, description: str, note: str = "") -> None: + write_text(self.directory / rel, text) + self.files.append((rel, layer, description, note)) + + +def stamp_now() -> str: + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def unique_stamp(out: Path) -> str: + stamp = stamp_now() + candidate, n = stamp, 1 + while (out / ("bugreport-" + candidate)).exists() or (out / ("bugreport-%s.zip" % candidate)).exists(): + n += 1 + candidate = "%s-%d" % (stamp, n) + return candidate + + +def build_manifest(args, root, stamp, bundle, wikitool, trace, gaps) -> str: + lines = ["# Bug report bundle", "", + "Collected: %s UTC " % datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S"), + "Bundle: `bugreport-%s`" % stamp, "", "## Privacy", "", PRIVACY_NOTICE, "", + "## Options", "", + "- chronology: %s" % ("supplied" if args.chronology else "not supplied"), + "- transcripts: %d" % len(args.transcript), + "- page titles (`--titles`): %s" % ("included" if args.titles else "kept out"), + "- session trace: %s" % ( + "excluded (`--no-trace`)" if args.no_trace else + ("included" if trace and trace.get("found") else "not found")), + "", "## wikitool", ""] + if wikitool is None: + lines.append("`wikitool` did not start: no launcher found under `tools/`.") + elif wikitool["started"]: + lines.append("`wikitool` started. Five read commands ran; outputs are verbatim.") + else: + lines.append("`wikitool` did not start (%s). The four other commands were not run." + % wikitool["reason"]) + if wikitool: + lines += ["", "| Command | Session id |", "|---|---|"] + for out in wikitool["outputs"]: + lines.append("| `%s` | %s |" % (out["label"], out["session"])) + caller, source = resolve_trace_session(args.session) + lines += ["", "Caller session id: %s" % ( + "%s (from %s)" % (caller, source) if caller else + "not set; `wikitool` falls back to its parent process id"), ""] + if trace and not trace.get("found") and not args.no_trace: + lines += ["No trace found for the caller's session%s. Existing session directories " + "(name and size only):" % ( + " `%s`" % trace["session"] if trace.get("session") else ""), ""] + for item in trace.get("listing", [])[:100]: + lines.append("- `%s` (%d bytes)" % (item["session_directory"], item["bytes"])) + if not trace.get("listing"): + lines.append("- none") + lines.append("") + lines += ["## Files", "", "| File | Layer | Contents | Note |", "|---|---|---|---|"] + for rel, layer, description, note in sorted(bundle.files, key=lambda f: f[0]): + lines.append("| `%s` | %s | %s | %s |" % (rel, layer, description, note)) + lines.append("| `MANIFEST.md` | - | this file | |") + if gaps: + lines += ["", "## Gaps", ""] + ["- %s" % gap for gap in gaps] + return "\n".join(lines) + "\n" + + +def final_pass(bundle_dir: Path, red: Redactor, shield, skip) -> None: + for path in sorted(bundle_dir.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(bundle_dir).as_posix() + text = read_text(path) + cleaned = red.scrub(text) + if shield is not None and rel not in skip: + cleaned = shield.apply(cleaned) + if cleaned != text: + write_text(path, cleaned) + + +def make_zip(bundle_dir: Path, archive: Path) -> None: + with zipfile.ZipFile(str(archive), "w", zipfile.ZIP_DEFLATED) as zf: + for path in sorted(bundle_dir.rglob("*")): + if path.is_file(): + zf.write(str(path), bundle_dir.name + "/" + path.relative_to(bundle_dir).as_posix()) + + +def parse_args(argv): + parser = argparse.ArgumentParser( + prog="bugreport.py", + description="Collect a bug-report bundle. Removes secrets, keeps page titles out " + "unless --titles is given, uploads nothing.", + ) + parser.add_argument("--chronology", help="the agent's fact-only chronology (Markdown)") + parser.add_argument("--transcript", action="append", default=[], + help="a harness transcript to include (repeatable; only on request)") + group = parser.add_mutually_exclusive_group() + group.add_argument("--session", help="session id whose trace to include") + group.add_argument("--no-trace", action="store_true", help="leave the session trace out") + parser.add_argument("--titles", action="store_true", + help="keep page titles and paths (writes tree-paths.txt)") + parser.add_argument("--root", help="checkout root (default: parent of tools/)") + parser.add_argument("--out", help="output directory (default: <root>/reports)") + return parser.parse_args(argv) + + +def main(argv=None) -> int: + args = parse_args(argv) + root = Path(args.root).resolve() if args.root else Path(__file__).resolve().parent.parent + out = Path(args.out).resolve() if args.out else root / "reports" + + inputs = [("chronology", args.chronology)] + [("transcript", t) for t in args.transcript] + for label, value in inputs: + if value and not Path(value).is_file(): + sys.stderr.write("bugreport: %s file not found: %s\n" % (label, value)) + return 1 + + try: + out.mkdir(parents=True, exist_ok=True) + stamp = unique_stamp(out) + bundle_dir = out / ("bugreport-" + stamp) + bundle_dir.mkdir() + return _collect(args, root, out, stamp, bundle_dir) + except OSError as exc: + sys.stderr.write("bugreport: cannot write the bundle: %s\n" % exc) + return 1 + + +def _collect(args, root: Path, out: Path, stamp: str, bundle_dir: Path) -> int: + red = Redactor() + bundle = Bundle(bundle_dir) + gaps = [] + + trace = None + if not args.no_trace: + trace = guarded(collect_trace, root, args.session) + if trace.get("found"): + bundle.add("trace.jsonl", trace["text"], "3", "session trace of the caller", + MAY_CONTAIN_CONTENT) + + bundle.add("environment.json", dump_json(guarded(collect_environment, root, red)), "1", + "OS, Python, shells, harness, PATH, environment variables (names; values for a " + "fixed list), git configuration, venv, line endings, Windows details") + + stack = guarded(collect_stack, root, red) + bundle.add("stack.json", dump_json(stack), "2", + "VERSION, `.wikitool-*.json` (secrets removed), git status and log") + + content_paths = [] + tree = {} + for name in CONTENT_DIRS: + summary, files = _safe_scan(root, name) + tree[name] = summary + content_paths.extend(files) + bundle.add("tree-structure.json", dump_json(tree), "2", + "kb/ and raw/: counts, depth, path lengths, name problems - no content") + if args.titles: + bundle.add("tree-paths.txt", "\n".join(sorted(content_paths)) + "\n", "2", + "all relative paths under kb/ and raw/", "contains page titles") + + wikitool = None + launcher = launcher_command(root) + if launcher: + wikitool = run_wikitool(root, launcher, "bugreport-" + stamp) + for item in wikitool["outputs"]: + bundle.add(item["file"], item["text"], "3", "verbatim output of `%s`" % item["label"]) + else: + gaps.append("no launcher found under tools/, so no wikitool command ran") + + if args.chronology: + bundle.add("CHRONOLOGY.md", read_text(Path(args.chronology)), "4", + "the agent's chronology", MAY_CONTAIN_CONTENT) + else: + gaps.append("no chronology was supplied") + used = set() + for source in args.transcript: + name = Path(source).name + target, n = name, 1 + while target in used: + n += 1 + target = "%d-%s" % (n, name) + used.add(target) + bundle.add("transcripts/" + target, read_text(Path(source)), "4", + "harness transcript", MAY_CONTAIN_CONTENT) + + write_text(bundle_dir / "MANIFEST.md", + build_manifest(args, root, stamp, bundle, wikitool, trace, gaps)) + + shield = None + if not args.titles: + shield = TitleShield( + rel for rel in content_paths if rel.rsplit("/", 1)[-1] not in STACK_NAMES + ) + final_pass(bundle_dir, red, shield, {"tree-paths.txt"}) + + archive = out / ("bugreport-%s.zip" % stamp) + make_zip(bundle_dir, archive) + + print("Bundle: %s" % bundle_dir) + print("Archive: %s" % archive) + print() + print(PRIVACY_NOTICE) + return 0 + + +def _safe_scan(root: Path, name: str): + try: + return scan_tree(root, name) + except Exception as exc: # noqa: BLE001 + return {"collector_error": "%s: %s" % (type(exc).__name__, exc)}, [] + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/chemenu/tests/test_bugreport.py b/tools/chemenu/tests/test_bugreport.py new file mode 100644 index 0000000..84b9456 --- /dev/null +++ b/tools/chemenu/tests/test_bugreport.py @@ -0,0 +1,327 @@ +"""`tools/bugreport.py`, the bug-report collector. + +The collector is a standalone script - standard library only, no `chemenu` import - +so these tests import it the way `test_trace_ingest.py` imports its script, and run +it against a small fake checkout with a fake `tools/wikitool` launcher. +""" +import ast +import json +import os +import subprocess +import sys +import zipfile +from pathlib import Path + +import pytest + +import bugreport + +SCRIPT = Path(bugreport.__file__) + +TOKEN = "s3cr3t-tok3n-value-9F2" +PASSWORD = "hunter2-hunter2" +TITLE = "Vertraulicher Titel Alpha" +OTHER_TITLE = "Zweite Seite Beta" + +LAUNCHER = """#!{python} +import os, sys +args = sys.argv[1:] +print("args:", " ".join(args)) +print("session:", os.environ.get("WIKITOOL_SESSION_ID", "<unset>")) +print("token-in-env:", os.environ.get("MY_API_TOKEN", "<unset>")) +print("see [[{title}]] and kb/Concepts/{title}.md and kb/Concepts/CONTRACT.md") +if args[:2] == ["version", "show"] and os.environ.get("FAKE_NOT_STARTING"): + sys.stderr.write("ModuleNotFoundError: No module named 'typer'\\n") + sys.exit(1) +if os.environ.get("FAKE_SLEEP") and args[0] == "doctor": + import time + time.sleep(30) +""" + + +def git(root: Path, *args: str) -> None: + subprocess.run( + ["git", "-c", "user.name=Fixture", "-c", "user.email=f@example.invalid", *args], + cwd=root, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + + +@pytest.fixture +def checkout(tmp_path: Path, monkeypatch) -> Path: + root = tmp_path / "checkout" + (root / "kb" / "Concepts").mkdir(parents=True) + (root / "raw" / "articles").mkdir(parents=True) + (root / "tools").mkdir() + (root / "kb" / "Concepts" / f"{TITLE}.md").write_text("body of the page\n") + (root / "kb" / "Concepts" / f"{OTHER_TITLE}.md").write_text("more\n") + (root / "kb" / "Concepts" / "CONTRACT.md").write_text("stack file\n") + (root / "raw" / "articles" / "Quelle Gamma.md").write_text("source\n") + (root / "VERSION").write_text("1.2.3\n") + (root / ".wikitool-upload.json").write_text( + json.dumps({"schema": 1, "api_token": TOKEN, "enabled": True}) + ) + (root / ".wikitool-tools.json").write_text( + json.dumps({"schema": 1, "tools": {"python": {"path": "/definitely/not/here/python"}}}) + ) + launcher = root / "tools" / "wikitool" + launcher.write_text(LAUNCHER.format(python=sys.executable, title=TITLE)) + launcher.chmod(0o755) + git(root, "init", "-q", "-b", "main") + git(root, "remote", "add", "origin", f"https://oauth:{PASSWORD}@git.example.invalid/x/y.git") + git(root, "add", "-A") + git(root, "commit", "-q", "-m", "seed") + (root / "kb" / "Concepts" / f"{TITLE}.md").write_text("changed\n") + monkeypatch.delenv("WIKI_TRACE_DIR", raising=False) + monkeypatch.setenv("MY_API_TOKEN", TOKEN) + monkeypatch.setenv("PATH", os.environ["PATH"]) + return root + + +def collect(root: Path, tmp_path: Path, *extra: str) -> Path: + out = tmp_path / "out" + code = bugreport.main(["--root", str(root), "--out", str(out), *extra]) + assert code == 0 + bundles = [p for p in out.iterdir() if p.is_dir()] + assert len(bundles) == 1 + return bundles[0] + + +def all_text(bundle: Path) -> str: + return "\n".join( + p.read_text(encoding="utf-8", errors="replace") for p in bundle.rglob("*") if p.is_file() + ) + + +def test_the_script_is_standard_library_only_and_python38_syntax(): + source = SCRIPT.read_text(encoding="utf-8") + tree = ast.parse(source, feature_version=(3, 8)) + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module.split(".")[0]) + assert "chemenu" not in imported + assert imported <= set(sys.stdlib_module_names) if hasattr(sys, "stdlib_module_names") else True + + +def test_the_script_starts_on_a_bare_interpreter(): + proc = subprocess.run( + [sys.executable, "-I", "-S", str(SCRIPT), "--help"], capture_output=True, text=True + ) + assert proc.returncode == 0 + assert "--no-trace" in proc.stdout + + +def test_a_bundle_and_an_archive_are_written_with_a_manifest(checkout, tmp_path): + bundle = collect(checkout, tmp_path, "--no-trace") + for name in ("MANIFEST.md", "environment.json", "stack.json", "tree-structure.json"): + assert (bundle / name).is_file(), name + assert (bundle / "wikitool" / "doctor.txt").is_file() + manifest = (bundle / "MANIFEST.md").read_text() + assert "not pseudonymised" in manifest + assert "no chronology was supplied" in manifest + archive = bundle.with_name(bundle.name + ".zip") + with zipfile.ZipFile(archive) as zf: + names = zf.namelist() + assert f"{bundle.name}/MANIFEST.md" in names + + +def test_secrets_are_removed_everywhere_including_the_archive(checkout, tmp_path): + bundle = collect(checkout, tmp_path, "--no-trace") + text = all_text(bundle) + assert TOKEN not in text + assert PASSWORD not in text + stack = json.loads((bundle / "stack.json").read_text()) + assert stack["config_files"][".wikitool-upload.json"]["api_token"] == "<removed>" + archive = bundle.with_name(bundle.name + ".zip") + with zipfile.ZipFile(archive) as zf: + for name in zf.namelist(): + data = zf.read(name).decode("utf-8", errors="replace") + assert TOKEN not in data and PASSWORD not in data, name + + +def test_a_secret_from_a_config_file_is_also_scrubbed_from_a_chronology(checkout, tmp_path): + chron = tmp_path / "chron.md" + chron.write_text(f"1. Setup failed with {TOKEN} in the header\n") + bundle = collect(checkout, tmp_path, "--no-trace", "--chronology", str(chron)) + assert TOKEN not in (bundle / "CHRONOLOGY.md").read_text() + assert "1. Setup failed" in (bundle / "CHRONOLOGY.md").read_text() + + +def test_environment_records_every_name_but_values_only_for_the_allowlist( + checkout, tmp_path, monkeypatch +): + monkeypatch.setenv("SOMETHING_PRIVATE", "very-private-value") + monkeypatch.setenv("WIKI_TRACE", "1") + monkeypatch.setenv("WIKITOOL_UPDATE_TOKEN", "allowlisted-but-secret") + bundle = collect(checkout, tmp_path, "--no-trace") + env = json.loads((bundle / "environment.json").read_text())["env"] + assert env["SOMETHING_PRIVATE"] == "<not collected>" + assert env["MY_API_TOKEN"] == "<not collected>" + assert env["WIKI_TRACE"] == "1" + assert env["WIKITOOL_UPDATE_TOKEN"] == "<removed>" + assert "PATH" in env and env["PATH"] == os.environ["PATH"] + text = all_text(bundle) + assert "very-private-value" not in text + assert "allowlisted-but-secret" not in text + + +def test_page_titles_are_kept_out_by_default_and_kept_with_titles(checkout, tmp_path): + bundle = collect(checkout, tmp_path, "--no-trace") + text = all_text(bundle) + assert TITLE not in text + assert OTHER_TITLE not in text + assert "Quelle Gamma" not in text + assert not (bundle / "tree-paths.txt").exists() + assert "kb/Concepts/CONTRACT.md" in text # stack-generated names are not titles + + kept = collect(checkout, tmp_path / "second", "--no-trace", "--titles") + assert TITLE in all_text(kept) + assert TITLE in (kept / "tree-paths.txt").read_text() + + +def test_wikilinks_in_captured_output_lose_their_title(checkout, tmp_path): + bundle = collect(checkout, tmp_path, "--no-trace") + doctor = (bundle / "wikitool" / "doctor.txt").read_text() + assert f"[[{TITLE}]]" not in doctor + assert "[[<title" in doctor + + +def test_counting_calls_run_under_the_collectors_own_session_inherited_ones_do_not( + checkout, tmp_path, monkeypatch +): + monkeypatch.setenv("WIKITOOL_SESSION_ID", "callers-session") + bundle = collect(checkout, tmp_path, "--no-trace") + own = f"bugreport-{bundle.name.split('bugreport-', 1)[1]}" + for name in ("instructions-verify", "docs-verify"): + assert f"session: {own}" in (bundle / "wikitool" / f"{name}.txt").read_text() + for name in ("version-show", "doctor", "budget-status"): + assert "session: callers-session" in (bundle / "wikitool" / f"{name}.txt").read_text() + manifest = (bundle / "MANIFEST.md").read_text() + assert "callers-session (from WIKITOOL_SESSION_ID)" in manifest + + +def test_the_callers_trace_is_copied_and_no_trace_leaves_it_out( + checkout, tmp_path, monkeypatch +): + monkeypatch.setenv("WIKITOOL_SESSION_ID", "callers-session") + trace_dir = checkout / "reports" / "telemetry" / "callers-session" + trace_dir.mkdir(parents=True) + (trace_dir / "trace.jsonl").write_text('{"event": "x"}\n') + other = checkout / "reports" / "telemetry" / "other-session" + other.mkdir() + (other / "trace.jsonl").write_text('{"event": "y"}\n') + + with_trace = collect(checkout, tmp_path / "a") + assert (with_trace / "trace.jsonl").read_text() == '{"event": "x"}\n' + assert "may contain page content and titles" in (with_trace / "MANIFEST.md").read_text() + + without = collect(checkout, tmp_path / "b", "--no-trace") + assert not (without / "trace.jsonl").exists() + + +def test_a_missing_trace_lists_the_sessions_that_exist(checkout, tmp_path, monkeypatch): + monkeypatch.setenv("WIKITOOL_SESSION_ID", "nobody-wrote-this") + other = checkout / "reports" / "telemetry" / "some-session" + other.mkdir(parents=True) + (other / "trace.jsonl").write_text("12345") + bundle = collect(checkout, tmp_path) + assert not (bundle / "trace.jsonl").exists() + manifest = (bundle / "MANIFEST.md").read_text() + assert "some-session" in manifest and "5 bytes" in manifest + + +def test_a_bugreport_bucket_is_never_offered_as_a_trace(checkout, monkeypatch): + monkeypatch.setenv("WIKITOOL_SESSION_ID", "missing") + base = checkout / "reports" / "telemetry" + (base / "bugreport-20260101T000000Z").mkdir(parents=True) + (base / "real-session").mkdir() + trace = bugreport.collect_trace(checkout, None) + assert [item["session_directory"] for item in trace["listing"]] == ["real-session"] + + +def test_a_wikitool_that_does_not_start_still_yields_a_report(checkout, tmp_path, monkeypatch): + monkeypatch.setenv("FAKE_NOT_STARTING", "1") + bundle = collect(checkout, tmp_path, "--no-trace") + assert (bundle / "wikitool" / "version-show.txt").is_file() + assert not (bundle / "wikitool" / "doctor.txt").exists() + assert "ModuleNotFoundError" in (bundle / "wikitool" / "version-show.txt").read_text() + manifest = (bundle / "MANIFEST.md").read_text() + assert "did not start" in manifest + assert (bundle / "environment.json").is_file() + + +def test_no_launcher_at_all_still_yields_a_report(checkout, tmp_path): + (checkout / "tools" / "wikitool").unlink() + bundle = collect(checkout, tmp_path, "--no-trace") + assert not (bundle / "wikitool").exists() + assert "no launcher" in (bundle / "MANIFEST.md").read_text() + + +def test_a_hanging_wikitool_call_is_cut_off(checkout, tmp_path, monkeypatch): + monkeypatch.setattr(bugreport, "TIMEOUT_SECONDS", 1) + monkeypatch.setenv("FAKE_SLEEP", "1") + bundle = collect(checkout, tmp_path, "--no-trace") + assert "timed out after 1 s" in (bundle / "wikitool" / "doctor.txt").read_text() + assert (bundle / "wikitool" / "docs-verify.txt").is_file() + + +def test_the_tools_config_path_check_reports_what_is_missing(checkout, tmp_path): + bundle = collect(checkout, tmp_path, "--no-trace") + config = json.loads((bundle / "environment.json").read_text())["tools_config"] + assert config["status"] == "present" + assert config["path_checks"] == [ + {"at": "tools.python.path", "path": "/definitely/not/here/python", "exists": False} + ] + + +def test_a_missing_chronology_file_is_an_error_and_writes_nothing(checkout, tmp_path): + out = tmp_path / "out" + code = bugreport.main( + ["--root", str(checkout), "--out", str(out), "--chronology", str(tmp_path / "nope.md")] + ) + assert code == 1 + assert not out.exists() or not list(out.iterdir()) + + +def test_session_and_no_trace_exclude_each_other(checkout, tmp_path): + with pytest.raises(SystemExit): + bugreport.main(["--root", str(checkout), "--session", "x", "--no-trace"]) + + +def test_transcripts_are_added_and_scrubbed(checkout, tmp_path): + transcript = tmp_path / "t.jsonl" + transcript.write_text(f'{{"msg": "token {TOKEN}"}}\n') + bundle = collect(checkout, tmp_path, "--no-trace", "--transcript", str(transcript)) + copied = (bundle / "transcripts" / "t.jsonl").read_text() + assert TOKEN not in copied + + +def test_two_bundles_in_the_same_second_do_not_collide(checkout, tmp_path, monkeypatch): + monkeypatch.setattr(bugreport, "stamp_now", lambda: "20260101T000000Z") + out = tmp_path / "out" + assert bugreport.main(["--root", str(checkout), "--out", str(out), "--no-trace"]) == 0 + assert bugreport.main(["--root", str(checkout), "--out", str(out), "--no-trace"]) == 0 + assert sorted(p.name for p in out.iterdir() if p.is_dir()) == [ + "bugreport-20260101T000000Z", "bugreport-20260101T000000Z-2", + ] + + +def test_the_collector_leaves_the_checkout_untouched(checkout, tmp_path): + def snapshot(): + return sorted( + (p.relative_to(checkout).as_posix(), p.stat().st_mtime_ns) + for p in checkout.rglob("*") + if p.is_file() and ".git/" not in p.as_posix() + ) + + before = snapshot() + collect(checkout, tmp_path, "--no-trace") + assert snapshot() == before + + +def test_the_collector_ships_with_a_distribution(): + from chemenu.commands import dist_cmd + + assert "tools/bugreport.py" in dist_cmd.build_plan()