diff --git a/.gitignore b/.gitignore index d71ea5f..aa940b7 100644 --- a/.gitignore +++ b/.gitignore @@ -73,6 +73,11 @@ npm-debug.log* # "Gates") - local, per-session, never committed /tools/.wikitool_session/ +# `wikitool raw capture`/`raw status` git cache (see raw/CONTRACT.md "Getting a +# repository in") - one bare repository per captured URL, refetched on demand. +# Derived and per-checkout; never committed, never shipped. +/tools/.wikitool_capture/ + # Go /go.mod /go.sum diff --git a/CHANGES.md b/CHANGES.md index 385b29b..66ec2b5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -59,7 +59,7 @@ concern - readable here, never shipped as something to parse. --- -## 8.0.0-beta.40 - 2026-10-04 - Comparison and source pages accept the sources: that cite add writes; sources may cite sources +## 8.0.0-beta.41 - 2026-10-05 - raw capture / raw status / --replaces-bundle: documentation from git repositories as a bundle, with drift reporting **Author:** Torben Nehmer @@ -108,6 +108,7 @@ concern - readable here, never shipped as something to parse. - Organisationsseiten: Personen als Abschnitt mit Aufstieg, entity_type organization, member-of, Lint-Befund broken_anchors - lint: Unfilled Template Sections - a section still holding only its template's TODO placeholders (advisory) - Comparison and source pages accept the sources: that cite add writes; sources may cite sources +- raw capture / raw status / --replaces-bundle: documentation from git repositories as a bundle, with drift reporting **Low impact** - version bump no longer points at version release in its output @@ -153,6 +154,53 @@ concern - readable here, never shipped as something to parse. - wiki-ingest and wiki-manage call xref add with --rel, not the --rel-a/--rel-b removed in 4.0.0 +### raw capture / raw status / --replaces-bundle: documentation from git repositories as a bundle, with drift reporting + +Documentation from a git repository used to reach an instance only by hand: copy the files into +`incoming/`, rename them around the global name rule, and remember nowhere which repository and +commit they came from. Three commands replace that: + +- **`raw capture --ref --path ... --name --fidelity + --authority `** resolves the ref rule - a branch, or a tag pattern such as `v*` that takes the + newest matching tag by version order - to one commit. It fetches that commit by ref name, + shallowly, into a bare cache per URL under `tools/.wikitool_capture/` (gitignored, never + exported), and writes the files the globs select into `incoming//` at their repository + paths. Files are read as blobs, never through a checkout, so they are byte-identical to the + repository even under `core.autocrlf`. Globs follow git's `:(glob)` pathspec, checked against + git itself in the tests. `_capture.json` beside the files records repository, ref rule, commit, + globs, capture time, `fidelity`, `authority` and the file list. `--update ` captures + the current state from that manifest. +- **`raw status`** reports every captured bundle whose files changed in the repository, as + `A`/`M`/`D` grouped by owning source page, and stays quiet about a commit that moved without a + change inside the globs. An unreachable repository is one line, never an abort; `--json` is + for the coming intake run. +- **`raw accept --replaces-bundle incoming/`** replaces a captured bundle as + a whole at its existing address. Afterwards it holds exactly the new manifest's files plus + `_capture.json`; files the repository dropped are removed, and emptied directories `rmdir`ed. + `raw_files:` is left alone, as with `--replaces`, and the command prints the `touch --add/--remove + raw_files=` lines that follow. + +Mechanical exclusions, each named in the output: a file whose first line starts with +`\n# Exported\n") + repo.write(".github/workflow.md", "# CI\n") + repo.write("docs/big.bin", "version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 9\n") + repo.write("docs/sub/_capture.json", "{}\n") + os.symlink("a.md", repo.path / "docs" / "link.md") + sub = repo.commit() + _git(repo.path, "update-index", "--add", "--cacheinfo", f"160000,{sub},vendor/lib") + _git(repo.path, "commit", "-q", "-m", "submodule") + + _capture(repo.url, paths=("**",)) + out = _out(capsys) + for path, reason in ( + ("docs/guideline.md", "guideline export"), + (".github/workflow.md", "hidden path segment"), + ("docs/big.bin", "Git LFS pointer"), + ("docs/sub/_capture.json", "reserved name"), + ("docs/link.md", "symlink"), + ("vendor/lib", "submodule"), + ): + assert f"excluded {path} ({reason}" in out + files = json.loads((capture / "incoming" / "svc" / "_capture.json").read_text(encoding="utf-8"))["files"] + assert files == ["README.md", "docs/a.md", "docs/config.yaml", "docs/x/y/b.md", "src/main.py"] + + _accept(capture / "incoming" / "svc") + repo.write("docs/a.md", "# A, changed\n") + repo.commit() + capsys.readouterr() + raw_status_command(json_out=True) + [row] = json.loads(capsys.readouterr().out) + assert row["files"] == [{"path": f"{_shard()}/svc/docs/a.md", "status": "M"}] + + +def test_a_file_over_the_size_limit_is_excluded(capture, repo, monkeypatch, capsys): + from chemenu import web_capture + + monkeypatch.setattr(web_capture, "MAX_BYTES", 5) + _capture(repo.url, paths=("docs/x/**",)) + assert "excluded docs/x/y/b.md (over" not in _out(capsys) # 4 bytes: under + with pytest.raises(typer.Exit): + _capture(repo.url, paths=("README.md",), name="svc2") + assert "excluded README.md (over" in _out(capsys) + + +# --- URLs and credentials ----------------------------------------------------- + + +@pytest.mark.parametrize("url", [ + "ext::sh -c touch% /tmp/x", + "fd::17", + "file:///srv/repo.git", + "/srv/repo.git", + "http://example.org/repo.git", + "https://user:token@example.org/repo.git", + "ssh://user:secret@example.org/repo.git", + "-uhelp@example.org:x", +]) +def test_refused_urls(url): + with pytest.raises(ValidationError): + repo_capture.check_repo_url(url) + + +@pytest.mark.parametrize("url", [ + "https://example.org/team/repo.git", + "https://user@example.org/team/repo.git", + "ssh://git@example.org:2222/team/repo.git", + "git@example.org:team/repo.git", +]) +def test_accepted_urls(url): + repo_capture.check_repo_url(url) + + +def _ext_url(marker: Path) -> str: + return f"ext::sh -c touch% {marker}" + + +@posix_only +def test_capture_refuses_an_ext_url_and_runs_nothing(capture, tmp_path): + marker = tmp_path / "ext-ran" + with pytest.raises(typer.Exit): + _capture(_ext_url(marker)) + assert not marker.exists() + + +@posix_only +def test_git_itself_refuses_the_ext_transport(capture, tmp_path): + """Below the URL check: the git layer alone, handed the URL directly.""" + marker = tmp_path / "ext-ran" + with pytest.raises(BackendError): + with repo_capture.cache_repo("ext-probe") as cache: + repo_capture.resolve_ref(_ext_url(marker), "main", cache) + assert not marker.exists() + + +@posix_only +def test_the_ext_probe_would_run_if_git_allowed_it(capture, tmp_path, monkeypatch): + """The control for the two tests above: the same URL does run its command + once `ext` is allowed, so their empty marker means something.""" + marker = tmp_path / "ext-ran" + monkeypatch.setattr(repo_capture, "ALLOWED_SCHEMES", ("ext",)) + with pytest.raises(BackendError): + with repo_capture.cache_repo("ext-probe") as cache: + repo_capture.resolve_ref(_ext_url(marker), "main", cache) + assert marker.exists() + + +def test_a_refused_url_leaves_no_credential_anywhere(capture, tmp_path): + with pytest.raises(typer.Exit): + _capture("https://user:s3cr3t-token@example.org/repo.git") + for path in tmp_path.rglob("*"): + if path.is_file(): + assert b"s3cr3t-token" not in path.read_bytes(), path + + +def _plant_bundle(capture, name, url, commit="0" * 40, files=("README.md",)): + """A captured bundle under raw/ as an earlier accept would have left it.""" + bundle = capture / _shard() / name + bundle.mkdir(parents=True) + for rel in files: + (bundle / rel).write_text("planted\n", encoding="utf-8") + manifest = { + "schema": 1, "repo": url, "ref": "main", "commit": commit, "paths": ["README.md"], + "captured": "2026-10-01T00:00:00Z", "fidelity": "verbatim", "authority": "normative", + "files": list(files), + } + (bundle / "_capture.json").write_text(json.dumps(manifest), encoding="utf-8") + return bundle + + +@posix_only +def test_update_and_status_refuse_an_ext_url_read_from_a_manifest(capture, tmp_path, capsys): + marker = tmp_path / "ext-ran" + bundle = _plant_bundle(capture, "evil", _ext_url(marker)) + with pytest.raises(typer.Exit): + _update(bundle) + capsys.readouterr() + raw_status_command(json_out=True) + [row] = json.loads(capsys.readouterr().out) + assert "refused" in row["error"] + assert not marker.exists() + + +@pytest.fixture +def asks_for_credentials(monkeypatch): + """An HTTP server that answers every request with a Basic-auth challenge - + the repository that would make an unguarded git prompt for a password.""" + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 - http.server's name + self.send_response(401) + self.send_header("WWW-Authenticate", 'Basic realm="repo"') + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + monkeypatch.setattr(repo_capture, "ALLOWED_SCHEMES", ("ssh", "https", "file", "http")) + yield f"http://127.0.0.1:{server.server_address[1]}/repo.git" + server.shutdown() + + +def test_a_repository_asking_for_credentials_is_unreachable_not_a_hang(capture, asks_for_credentials, capsys): + _plant_bundle(capture, "locked", asks_for_credentials) + started = time.monotonic() + raw_status_command(json_out=True) + [row] = json.loads(capsys.readouterr().out) + assert row["error"].startswith("not reachable") + with pytest.raises(typer.Exit): + _capture(asks_for_credentials, name="locked2") + assert time.monotonic() - started < 30 + + +# --- raw status --------------------------------------------------------------- + + +def test_status_reports_changed_docs_and_goes_quiet_after_the_replacement(capture, repo, capsys): + bundle = _captured_and_accepted(capture, repo) + repo.write("docs/a.md", "# A, second edition\n") + repo.remove("docs/x/y/b.md") + repo.write("docs/new.md", "# New\n") + repo.commit() + capsys.readouterr() + + raw_status_command(json_out=True) + [row] = json.loads(capsys.readouterr().out) + prefix = f"{_shard()}/svc" + assert row["changed"] is True and row["error"] is None + assert row["new"] == _git(repo.path, "rev-parse", "HEAD") + assert row["files"] == [ + {"path": f"{prefix}/docs/a.md", "status": "M"}, + {"path": f"{prefix}/docs/new.md", "status": "A"}, + {"path": f"{prefix}/docs/x/y/b.md", "status": "D"}, + ] + + raw_status_command(json_out=False) + out = _out(capsys) + assert f"raw capture --update {prefix}" in out + assert f"--replaces-bundle {prefix}" in out + + _update(bundle) + _accept(capture / "incoming" / "svc", replaces_bundle=bundle) + capsys.readouterr() + raw_status_command(json_out=True) + [row] = json.loads(capsys.readouterr().out) + assert row["changed"] is False and row["files"] == [] + + +def test_status_ignores_a_change_outside_the_globs(capture, repo, capsys): + _captured_and_accepted(capture, repo) + repo.write("src/main.py", "print('changed')\n") + repo.commit() + capsys.readouterr() + raw_status_command(json_out=True) + [row] = json.loads(capsys.readouterr().out) + assert row["old"] != row["new"] and row["changed"] is False + raw_status_command(json_out=False) + out = _out(capsys) + assert "1 unchanged" in out and "svc" not in out.split("raw/:")[1].replace("1 unchanged.", "") + + +def test_status_follows_new_tags_only_not_new_commits_on_main(capture, repo, capsys): + repo.tag("v1.0") + _captured_and_accepted(capture, repo, ref="v*") + repo.write("docs/a.md", "# A on main\n") + repo.commit() + capsys.readouterr() + raw_status_command(json_out=True) + assert json.loads(capsys.readouterr().out)[0]["changed"] is False + repo.tag("v1.1") + raw_status_command(json_out=True) + [row] = json.loads(capsys.readouterr().out) + assert row["changed"] is True + assert row["files"] == [{"path": f"{_shard()}/svc/docs/a.md", "status": "M"}] + + +def test_an_unreachable_repository_is_one_line_beside_the_others(capture, repo, tmp_path, capsys): + _captured_and_accepted(capture, repo) + _plant_bundle(capture, "gone", (tmp_path / "does-not-exist").as_uri()) + repo.write("docs/a.md", "# A, changed\n") + repo.commit() + capsys.readouterr() + raw_status_command(json_out=True) + rows = {Path(r["bundle"]).name: r for r in json.loads(capsys.readouterr().out)} + assert rows["gone"]["error"].startswith("not reachable") + assert rows["svc"]["changed"] is True and rows["svc"]["error"] is None + raw_status_command(json_out=False) # exits normally, both on their own line + out = capsys.readouterr().out + assert "not reachable" in out and "docs/a.md" in out + + +def test_status_with_no_captured_bundle(capture, capsys): + raw_status_command(json_out=False) + assert "No captured bundle" in capsys.readouterr().out + + +# --- raw accept --replaces-bundle --------------------------------------------- + + +def test_replaces_bundle_leaves_exactly_the_new_edition_in_place(capture, repo, kb_dir, capsys): + bundle = _captured_and_accepted(capture, repo) + prefix = f"{_shard()}/svc" + owned = [f"{prefix}/README.md", f"{prefix}/docs/a.md", f"{prefix}/docs/x/y/b.md"] + _write_source(kb_dir, "Source - Svc", owned) + + repo.write("docs/a.md", "# A, second edition\n") + repo.remove("docs/x/y/b.md") + repo.write("docs/new.md", "# New\n") + new_commit = repo.commit() + _update(bundle) + capsys.readouterr() + _accept(capture / "incoming" / "svc", replaces_bundle=bundle) + out = _out(capsys) + + on_disk = sorted(p.relative_to(bundle).as_posix() for p in bundle.rglob("*") if p.is_file()) + assert on_disk == ["README.md", "_capture.json", "docs/a.md", "docs/new.md"] + for rel in ("README.md", "docs/a.md", "docs/new.md"): + assert (bundle / rel).read_bytes() == (repo.path / rel).read_bytes() + assert not (bundle / "docs" / "x").exists() + assert json.loads((bundle / "_capture.json").read_text(encoding="utf-8"))["commit"] == new_commit + assert not (capture / "incoming" / "svc").exists() + assert read_page(kb_dir / "sources" / "Source - Svc.md")[0]["raw_files"] == owned + + assert f"M {prefix}/docs/a.md" in out and f"D {prefix}/docs/x/y/b.md" in out + assert f"A {prefix}/docs/new.md" in out + assert f'touch --page "Source - Svc" --remove raw_files={prefix}/docs/x/y/b.md' in out + assert f'touch --page "Source - Svc" --add raw_files={prefix}/docs/new.md' in out + + +def test_replaces_bundle_overwrites_changed_capture_fields_on_the_owning_page(capture, repo, kb_dir): + bundle = _captured_and_accepted(capture, repo) + _write_source(kb_dir, "Source - Svc", [f"{_shard()}/svc/README.md"]) + _update(bundle, authority="reporting") + _accept(capture / "incoming" / "svc", replaces_bundle=bundle) + assert read_page(kb_dir / "sources" / "Source - Svc.md")[0]["authority"] == "reporting" + + +def test_replaces_bundle_refuses_another_repository(capture, repo, tmp_path): + bundle = _captured_and_accepted(capture, repo) + other = Repo(tmp_path / "other") + other.write("README.md", "# Other\n") + other.commit() + _capture(other.url, paths=("README.md",), name="other") + (capture / "incoming" / "other").rename(capture / "incoming" / "svc") + before = _tree(capture / "incoming", capture / "raw") + with pytest.raises(typer.Exit): + _accept(capture / "incoming" / "svc", replaces_bundle=bundle) + assert _tree(capture / "incoming", capture / "raw") == before + + +def test_replaces_bundle_refuses_a_different_folder_name(capture, repo): + bundle = _captured_and_accepted(capture, repo) + _update(bundle) + (capture / "incoming" / "svc").rename(capture / "incoming" / "svc-renamed") + before = _tree(capture / "incoming", capture / "raw") + with pytest.raises(typer.Exit): + _accept(capture / "incoming" / "svc-renamed", replaces_bundle=bundle) + assert _tree(capture / "incoming", capture / "raw") == before + + +def test_replaces_bundle_refuses_a_bundle_without_a_manifest(capture, repo): + plain = capture / _shard() / "svc" + plain.mkdir(parents=True) + (plain / "README.md").write_text("hand-made folder bundle\n", encoding="utf-8") + _capture(repo.url, name="svc-new") + (capture / "incoming" / "svc-new").rename(capture / "incoming" / "svc") + before = _tree(capture / "incoming", capture / "raw") + with pytest.raises(typer.Exit): + _accept(capture / "incoming" / "svc", replaces_bundle=plain) + assert _tree(capture / "incoming", capture / "raw") == before + + +def test_replaces_bundle_refuses_page_replaces_and_capture_flags(capture, repo): + bundle = _captured_and_accepted(capture, repo) + _update(bundle) + before = _tree(capture / "incoming", capture / "raw") + for extra in ({"page": "Source - Svc"}, {"replaces": bundle / "README.md"}, {"fidelity": "verbatim"}): + with pytest.raises(typer.Exit): + _accept(capture / "incoming" / "svc", replaces_bundle=bundle, **extra) + assert _tree(capture / "incoming", capture / "raw") == before + + +def test_replaces_bundle_refuses_a_file_with_two_owners(capture, repo, kb_dir): + bundle = _captured_and_accepted(capture, repo) + readme = f"{_shard()}/svc/README.md" + _write_source(kb_dir, "Source - One", [readme]) + _write_source(kb_dir, "Source - Two", [readme]) + _update(bundle) + before = _tree(capture / "incoming", capture / "raw") + with pytest.raises(typer.Exit): + _accept(capture / "incoming" / "svc", replaces_bundle=bundle) + assert _tree(capture / "incoming", capture / "raw") == before + + +# --- --replaces / --page inside a captured bundle ----------------------------- + + +def test_replaces_refuses_a_target_inside_a_captured_bundle(capture, repo, capsys): + bundle = _captured_and_accepted(capture, repo) + (capture / "incoming" / "b.md").write_text("# B by hand\n", encoding="utf-8") + before = _tree(capture / "incoming", capture / "raw") + with pytest.raises(typer.Exit): + _accept(capture / "incoming" / "b.md", replaces=bundle / "docs" / "x" / "y" / "b.md") + assert _tree(capture / "incoming", capture / "raw") == before + assert "--replaces-bundle" in _out(capsys) + + +def test_page_refuses_to_grow_a_captured_bundle(capture, tmp_path, kb_dir): + flat = Repo(tmp_path / "flat") + flat.write("README.md", "# R\n") + flat.write("AGENTS.md", "# A\n") + flat.commit() + _capture(flat.url, paths=("*.md",), name="flat") + _accept(capture / "incoming" / "flat") + prefix = f"{_shard()}/flat" + _write_source(kb_dir, "Source - Flat", [f"{prefix}/AGENTS.md", f"{prefix}/README.md"]) + (capture / "incoming" / "notes.md").write_text("# notes\n", encoding="utf-8") + before = _tree(capture / "incoming", capture / "raw") + with pytest.raises(typer.Exit): + _accept(capture / "incoming" / "notes.md", fidelity="verbatim", authority="normative", + page="Source - Flat") + assert _tree(capture / "incoming", capture / "raw") == before + + +def test_a_loose_file_named_like_the_manifest_is_refused(capture): + (capture / "incoming" / "_capture.json").write_text("{}\n", encoding="utf-8") + with pytest.raises(typer.Exit): + _accept(capture / "incoming" / "_capture.json", fidelity="verbatim", authority="normative") + + +# --- coverage ----------------------------------------------------------------- + + +def test_coverage_skips_the_manifest_and_sees_a_bundled_contract(capture, tmp_path): + r = Repo(tmp_path / "with-contract") + r.write("README.md", "# R\n") + r.write("raw/CONTRACT.md", "# a repository's own stage contract\n") + r.commit() + _capture(r.url, paths=("README.md", "raw/CONTRACT.md"), name="wc") + _accept(capture / "incoming" / "wc") + (capture / "raw" / "CONTRACT.md").write_text("# the stage contract\n", encoding="utf-8") + + uncovered = uncovered_raw_files(config.RAW_DIR, load_kb_pages(config.KB_DIR)) + prefix = f"{_shard()}/wc" + assert f"{prefix}/raw/CONTRACT.md" in uncovered + assert f"{prefix}/README.md" in uncovered + assert f"{prefix}/_capture.json" not in uncovered + assert "raw/CONTRACT.md" not in uncovered