feat: bug-report collector tools/bugreport.py and instructions/bug-report.md (#157)
Files changed: - .gitea/workflows/ci.yml - CHANGES.md - INSTALL.md - VERSION - instructions/bug-report.md - instructions/gates.md - instructions/setup-instance.md - instructions/upgrade-instance.md - reports/CONTRACT.md - tools/README.md - tools/bugreport.py - tools/chemenu/tests/test_bugreport.py
This commit is contained in:
torben committed
1 parent
40413f966d
commit
f9c047bd2e
12 files changed
+1497
-3
No files matched your search
@@ -0,0 +1,327 @@
|
||||
"""`tools/bugreport.py`, the bug-report collector.
|
||||
|
||||
The collector is a standalone script - standard library only, no `chemenu` import -
|
||||
so these tests import it the way `test_trace_ingest.py` imports its script, and run
|
||||
it against a small fake checkout with a fake `tools/wikitool` launcher.
|
||||
"""
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import bugreport
|
||||
|
||||
SCRIPT = Path(bugreport.__file__)
|
||||
|
||||
TOKEN = "s3cr3t-tok3n-value-9F2"
|
||||
PASSWORD = "hunter2-hunter2"
|
||||
TITLE = "Vertraulicher Titel Alpha"
|
||||
OTHER_TITLE = "Zweite Seite Beta"
|
||||
|
||||
LAUNCHER = """#!{python}
|
||||
import os, sys
|
||||
args = sys.argv[1:]
|
||||
print("args:", " ".join(args))
|
||||
print("session:", os.environ.get("WIKITOOL_SESSION_ID", "<unset>"))
|
||||
print("token-in-env:", os.environ.get("MY_API_TOKEN", "<unset>"))
|
||||
print("see [[{title}]] and kb/Concepts/{title}.md and kb/Concepts/CONTRACT.md")
|
||||
if args[:2] == ["version", "show"] and os.environ.get("FAKE_NOT_STARTING"):
|
||||
sys.stderr.write("ModuleNotFoundError: No module named 'typer'\\n")
|
||||
sys.exit(1)
|
||||
if os.environ.get("FAKE_SLEEP") and args[0] == "doctor":
|
||||
import time
|
||||
time.sleep(30)
|
||||
"""
|
||||
|
||||
|
||||
def git(root: Path, *args: str) -> None:
|
||||
subprocess.run(
|
||||
["git", "-c", "user.name=Fixture", "-c", "user.email=f@example.invalid", *args],
|
||||
cwd=root, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def checkout(tmp_path: Path, monkeypatch) -> Path:
|
||||
root = tmp_path / "checkout"
|
||||
(root / "kb" / "Concepts").mkdir(parents=True)
|
||||
(root / "raw" / "articles").mkdir(parents=True)
|
||||
(root / "tools").mkdir()
|
||||
(root / "kb" / "Concepts" / f"{TITLE}.md").write_text("body of the page\n")
|
||||
(root / "kb" / "Concepts" / f"{OTHER_TITLE}.md").write_text("more\n")
|
||||
(root / "kb" / "Concepts" / "CONTRACT.md").write_text("stack file\n")
|
||||
(root / "raw" / "articles" / "Quelle Gamma.md").write_text("source\n")
|
||||
(root / "VERSION").write_text("1.2.3\n")
|
||||
(root / ".wikitool-upload.json").write_text(
|
||||
json.dumps({"schema": 1, "api_token": TOKEN, "enabled": True})
|
||||
)
|
||||
(root / ".wikitool-tools.json").write_text(
|
||||
json.dumps({"schema": 1, "tools": {"python": {"path": "/definitely/not/here/python"}}})
|
||||
)
|
||||
launcher = root / "tools" / "wikitool"
|
||||
launcher.write_text(LAUNCHER.format(python=sys.executable, title=TITLE))
|
||||
launcher.chmod(0o755)
|
||||
git(root, "init", "-q", "-b", "main")
|
||||
git(root, "remote", "add", "origin", f"https://oauth:{PASSWORD}@git.example.invalid/x/y.git")
|
||||
git(root, "add", "-A")
|
||||
git(root, "commit", "-q", "-m", "seed")
|
||||
(root / "kb" / "Concepts" / f"{TITLE}.md").write_text("changed\n")
|
||||
monkeypatch.delenv("WIKI_TRACE_DIR", raising=False)
|
||||
monkeypatch.setenv("MY_API_TOKEN", TOKEN)
|
||||
monkeypatch.setenv("PATH", os.environ["PATH"])
|
||||
return root
|
||||
|
||||
|
||||
def collect(root: Path, tmp_path: Path, *extra: str) -> Path:
|
||||
out = tmp_path / "out"
|
||||
code = bugreport.main(["--root", str(root), "--out", str(out), *extra])
|
||||
assert code == 0
|
||||
bundles = [p for p in out.iterdir() if p.is_dir()]
|
||||
assert len(bundles) == 1
|
||||
return bundles[0]
|
||||
|
||||
|
||||
def all_text(bundle: Path) -> str:
|
||||
return "\n".join(
|
||||
p.read_text(encoding="utf-8", errors="replace") for p in bundle.rglob("*") if p.is_file()
|
||||
)
|
||||
|
||||
|
||||
def test_the_script_is_standard_library_only_and_python38_syntax():
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
tree = ast.parse(source, feature_version=(3, 8))
|
||||
imported = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
imported.update(alias.name.split(".")[0] for alias in node.names)
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
imported.add(node.module.split(".")[0])
|
||||
assert "chemenu" not in imported
|
||||
assert imported <= set(sys.stdlib_module_names) if hasattr(sys, "stdlib_module_names") else True
|
||||
|
||||
|
||||
def test_the_script_starts_on_a_bare_interpreter():
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "-I", "-S", str(SCRIPT), "--help"], capture_output=True, text=True
|
||||
)
|
||||
assert proc.returncode == 0
|
||||
assert "--no-trace" in proc.stdout
|
||||
|
||||
|
||||
def test_a_bundle_and_an_archive_are_written_with_a_manifest(checkout, tmp_path):
|
||||
bundle = collect(checkout, tmp_path, "--no-trace")
|
||||
for name in ("MANIFEST.md", "environment.json", "stack.json", "tree-structure.json"):
|
||||
assert (bundle / name).is_file(), name
|
||||
assert (bundle / "wikitool" / "doctor.txt").is_file()
|
||||
manifest = (bundle / "MANIFEST.md").read_text()
|
||||
assert "not pseudonymised" in manifest
|
||||
assert "no chronology was supplied" in manifest
|
||||
archive = bundle.with_name(bundle.name + ".zip")
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
names = zf.namelist()
|
||||
assert f"{bundle.name}/MANIFEST.md" in names
|
||||
|
||||
|
||||
def test_secrets_are_removed_everywhere_including_the_archive(checkout, tmp_path):
|
||||
bundle = collect(checkout, tmp_path, "--no-trace")
|
||||
text = all_text(bundle)
|
||||
assert TOKEN not in text
|
||||
assert PASSWORD not in text
|
||||
stack = json.loads((bundle / "stack.json").read_text())
|
||||
assert stack["config_files"][".wikitool-upload.json"]["api_token"] == "<removed>"
|
||||
archive = bundle.with_name(bundle.name + ".zip")
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
for name in zf.namelist():
|
||||
data = zf.read(name).decode("utf-8", errors="replace")
|
||||
assert TOKEN not in data and PASSWORD not in data, name
|
||||
|
||||
|
||||
def test_a_secret_from_a_config_file_is_also_scrubbed_from_a_chronology(checkout, tmp_path):
|
||||
chron = tmp_path / "chron.md"
|
||||
chron.write_text(f"1. Setup failed with {TOKEN} in the header\n")
|
||||
bundle = collect(checkout, tmp_path, "--no-trace", "--chronology", str(chron))
|
||||
assert TOKEN not in (bundle / "CHRONOLOGY.md").read_text()
|
||||
assert "1. Setup failed" in (bundle / "CHRONOLOGY.md").read_text()
|
||||
|
||||
|
||||
def test_environment_records_every_name_but_values_only_for_the_allowlist(
|
||||
checkout, tmp_path, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("SOMETHING_PRIVATE", "very-private-value")
|
||||
monkeypatch.setenv("WIKI_TRACE", "1")
|
||||
monkeypatch.setenv("WIKITOOL_UPDATE_TOKEN", "allowlisted-but-secret")
|
||||
bundle = collect(checkout, tmp_path, "--no-trace")
|
||||
env = json.loads((bundle / "environment.json").read_text())["env"]
|
||||
assert env["SOMETHING_PRIVATE"] == "<not collected>"
|
||||
assert env["MY_API_TOKEN"] == "<not collected>"
|
||||
assert env["WIKI_TRACE"] == "1"
|
||||
assert env["WIKITOOL_UPDATE_TOKEN"] == "<removed>"
|
||||
assert "PATH" in env and env["PATH"] == os.environ["PATH"]
|
||||
text = all_text(bundle)
|
||||
assert "very-private-value" not in text
|
||||
assert "allowlisted-but-secret" not in text
|
||||
|
||||
|
||||
def test_page_titles_are_kept_out_by_default_and_kept_with_titles(checkout, tmp_path):
|
||||
bundle = collect(checkout, tmp_path, "--no-trace")
|
||||
text = all_text(bundle)
|
||||
assert TITLE not in text
|
||||
assert OTHER_TITLE not in text
|
||||
assert "Quelle Gamma" not in text
|
||||
assert not (bundle / "tree-paths.txt").exists()
|
||||
assert "kb/Concepts/CONTRACT.md" in text # stack-generated names are not titles
|
||||
|
||||
kept = collect(checkout, tmp_path / "second", "--no-trace", "--titles")
|
||||
assert TITLE in all_text(kept)
|
||||
assert TITLE in (kept / "tree-paths.txt").read_text()
|
||||
|
||||
|
||||
def test_wikilinks_in_captured_output_lose_their_title(checkout, tmp_path):
|
||||
bundle = collect(checkout, tmp_path, "--no-trace")
|
||||
doctor = (bundle / "wikitool" / "doctor.txt").read_text()
|
||||
assert f"[[{TITLE}]]" not in doctor
|
||||
assert "[[<title" in doctor
|
||||
|
||||
|
||||
def test_counting_calls_run_under_the_collectors_own_session_inherited_ones_do_not(
|
||||
checkout, tmp_path, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("WIKITOOL_SESSION_ID", "callers-session")
|
||||
bundle = collect(checkout, tmp_path, "--no-trace")
|
||||
own = f"bugreport-{bundle.name.split('bugreport-', 1)[1]}"
|
||||
for name in ("instructions-verify", "docs-verify"):
|
||||
assert f"session: {own}" in (bundle / "wikitool" / f"{name}.txt").read_text()
|
||||
for name in ("version-show", "doctor", "budget-status"):
|
||||
assert "session: callers-session" in (bundle / "wikitool" / f"{name}.txt").read_text()
|
||||
manifest = (bundle / "MANIFEST.md").read_text()
|
||||
assert "callers-session (from WIKITOOL_SESSION_ID)" in manifest
|
||||
|
||||
|
||||
def test_the_callers_trace_is_copied_and_no_trace_leaves_it_out(
|
||||
checkout, tmp_path, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("WIKITOOL_SESSION_ID", "callers-session")
|
||||
trace_dir = checkout / "reports" / "telemetry" / "callers-session"
|
||||
trace_dir.mkdir(parents=True)
|
||||
(trace_dir / "trace.jsonl").write_text('{"event": "x"}\n')
|
||||
other = checkout / "reports" / "telemetry" / "other-session"
|
||||
other.mkdir()
|
||||
(other / "trace.jsonl").write_text('{"event": "y"}\n')
|
||||
|
||||
with_trace = collect(checkout, tmp_path / "a")
|
||||
assert (with_trace / "trace.jsonl").read_text() == '{"event": "x"}\n'
|
||||
assert "may contain page content and titles" in (with_trace / "MANIFEST.md").read_text()
|
||||
|
||||
without = collect(checkout, tmp_path / "b", "--no-trace")
|
||||
assert not (without / "trace.jsonl").exists()
|
||||
|
||||
|
||||
def test_a_missing_trace_lists_the_sessions_that_exist(checkout, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("WIKITOOL_SESSION_ID", "nobody-wrote-this")
|
||||
other = checkout / "reports" / "telemetry" / "some-session"
|
||||
other.mkdir(parents=True)
|
||||
(other / "trace.jsonl").write_text("12345")
|
||||
bundle = collect(checkout, tmp_path)
|
||||
assert not (bundle / "trace.jsonl").exists()
|
||||
manifest = (bundle / "MANIFEST.md").read_text()
|
||||
assert "some-session" in manifest and "5 bytes" in manifest
|
||||
|
||||
|
||||
def test_a_bugreport_bucket_is_never_offered_as_a_trace(checkout, monkeypatch):
|
||||
monkeypatch.setenv("WIKITOOL_SESSION_ID", "missing")
|
||||
base = checkout / "reports" / "telemetry"
|
||||
(base / "bugreport-20260101T000000Z").mkdir(parents=True)
|
||||
(base / "real-session").mkdir()
|
||||
trace = bugreport.collect_trace(checkout, None)
|
||||
assert [item["session_directory"] for item in trace["listing"]] == ["real-session"]
|
||||
|
||||
|
||||
def test_a_wikitool_that_does_not_start_still_yields_a_report(checkout, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("FAKE_NOT_STARTING", "1")
|
||||
bundle = collect(checkout, tmp_path, "--no-trace")
|
||||
assert (bundle / "wikitool" / "version-show.txt").is_file()
|
||||
assert not (bundle / "wikitool" / "doctor.txt").exists()
|
||||
assert "ModuleNotFoundError" in (bundle / "wikitool" / "version-show.txt").read_text()
|
||||
manifest = (bundle / "MANIFEST.md").read_text()
|
||||
assert "did not start" in manifest
|
||||
assert (bundle / "environment.json").is_file()
|
||||
|
||||
|
||||
def test_no_launcher_at_all_still_yields_a_report(checkout, tmp_path):
|
||||
(checkout / "tools" / "wikitool").unlink()
|
||||
bundle = collect(checkout, tmp_path, "--no-trace")
|
||||
assert not (bundle / "wikitool").exists()
|
||||
assert "no launcher" in (bundle / "MANIFEST.md").read_text()
|
||||
|
||||
|
||||
def test_a_hanging_wikitool_call_is_cut_off(checkout, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(bugreport, "TIMEOUT_SECONDS", 1)
|
||||
monkeypatch.setenv("FAKE_SLEEP", "1")
|
||||
bundle = collect(checkout, tmp_path, "--no-trace")
|
||||
assert "timed out after 1 s" in (bundle / "wikitool" / "doctor.txt").read_text()
|
||||
assert (bundle / "wikitool" / "docs-verify.txt").is_file()
|
||||
|
||||
|
||||
def test_the_tools_config_path_check_reports_what_is_missing(checkout, tmp_path):
|
||||
bundle = collect(checkout, tmp_path, "--no-trace")
|
||||
config = json.loads((bundle / "environment.json").read_text())["tools_config"]
|
||||
assert config["status"] == "present"
|
||||
assert config["path_checks"] == [
|
||||
{"at": "tools.python.path", "path": "/definitely/not/here/python", "exists": False}
|
||||
]
|
||||
|
||||
|
||||
def test_a_missing_chronology_file_is_an_error_and_writes_nothing(checkout, tmp_path):
|
||||
out = tmp_path / "out"
|
||||
code = bugreport.main(
|
||||
["--root", str(checkout), "--out", str(out), "--chronology", str(tmp_path / "nope.md")]
|
||||
)
|
||||
assert code == 1
|
||||
assert not out.exists() or not list(out.iterdir())
|
||||
|
||||
|
||||
def test_session_and_no_trace_exclude_each_other(checkout, tmp_path):
|
||||
with pytest.raises(SystemExit):
|
||||
bugreport.main(["--root", str(checkout), "--session", "x", "--no-trace"])
|
||||
|
||||
|
||||
def test_transcripts_are_added_and_scrubbed(checkout, tmp_path):
|
||||
transcript = tmp_path / "t.jsonl"
|
||||
transcript.write_text(f'{{"msg": "token {TOKEN}"}}\n')
|
||||
bundle = collect(checkout, tmp_path, "--no-trace", "--transcript", str(transcript))
|
||||
copied = (bundle / "transcripts" / "t.jsonl").read_text()
|
||||
assert TOKEN not in copied
|
||||
|
||||
|
||||
def test_two_bundles_in_the_same_second_do_not_collide(checkout, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(bugreport, "stamp_now", lambda: "20260101T000000Z")
|
||||
out = tmp_path / "out"
|
||||
assert bugreport.main(["--root", str(checkout), "--out", str(out), "--no-trace"]) == 0
|
||||
assert bugreport.main(["--root", str(checkout), "--out", str(out), "--no-trace"]) == 0
|
||||
assert sorted(p.name for p in out.iterdir() if p.is_dir()) == [
|
||||
"bugreport-20260101T000000Z", "bugreport-20260101T000000Z-2",
|
||||
]
|
||||
|
||||
|
||||
def test_the_collector_leaves_the_checkout_untouched(checkout, tmp_path):
|
||||
def snapshot():
|
||||
return sorted(
|
||||
(p.relative_to(checkout).as_posix(), p.stat().st_mtime_ns)
|
||||
for p in checkout.rglob("*")
|
||||
if p.is_file() and ".git/" not in p.as_posix()
|
||||
)
|
||||
|
||||
before = snapshot()
|
||||
collect(checkout, tmp_path, "--no-trace")
|
||||
assert snapshot() == before
|
||||
|
||||
|
||||
def test_the_collector_ships_with_a_distribution():
|
||||
from chemenu.commands import dist_cmd
|
||||
|
||||
assert "tools/bugreport.py" in dist_cmd.build_plan()
|
||||
Reference in new issue
Block a user