From d4638bacdedaa959bce0cae787bd41843ec51da8 Mon Sep 17 00:00:00 2001 From: Torben Nehmer Date: Tue, 6 Oct 2026 08:41:29 +0200 Subject: [PATCH] fix: publish/sync merge generated files mechanically and carry non-overlapping uncommitted work through a rebase (#180) Overlap only in kb/index.md, kb/log.md, kb/provenance.md and kb/**/INDEX.md no longer fails a reconcile or reaches the rebase-review gate: the log keeps both sides' entries, the catalog and provenance are regenerated. Uncommitted work no incoming commit touches rides through the rebase via --autostash; the working tree is backed up under refs/wikitool/reconcile-backup first. is_generated is narrowed to kb/. Files changed: - CHANGES.md - VERSION - instructions/gates.md - instructions/session-setup.md - tools/CONTRACT.md - tools/chemenu/commands/git_publish.py - tools/chemenu/commands/index_build.py - tools/chemenu/commands/provenance_cmd.py - tools/chemenu/tests/test_git_publish.py Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2 --- CHANGES.md | 44 ++- VERSION | 2 +- instructions/gates.md | 7 +- instructions/session-setup.md | 6 +- tools/CONTRACT.md | 24 +- tools/chemenu/commands/git_publish.py | 435 +++++++++++++++++++++-- tools/chemenu/commands/index_build.py | 28 +- tools/chemenu/commands/provenance_cmd.py | 14 +- tools/chemenu/tests/test_git_publish.py | 405 ++++++++++++++++++++- 9 files changed, 907 insertions(+), 58 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f57ac32..b9257c0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -59,7 +59,7 @@ concern - readable here, never shipped as something to parse. --- -## 8.0.0-beta.45 - 2026-10-06 - kb/CONVENTIONS.md.template: the guideline filter is a default, not a setup question +## 8.0.0-beta.46 - 2026-10-06 - publish/sync merge generated files mechanically and carry non-overlapping uncommitted work through a rebase **Author:** Torben Nehmer @@ -111,6 +111,7 @@ concern - readable here, never shipped as something to parse. - raw capture / raw status / --replaces-bundle: documentation from git repositories as a bundle, with drift reporting - wiki-ingest: updating the captured repositories is a step-1 branch over raw status - export guidelines: the guideline pages as a generated GUIDELINES.md, pushed into the captured repositories behind the Guideline Push Gate +- publish/sync merge generated files mechanically and carry non-overlapping uncommitted work through a rebase **Low impact** - version bump no longer points at version release in its output @@ -158,6 +159,47 @@ concern - readable here, never shipped as something to parse. - kb/CONVENTIONS.md.template: the guideline filter is a default, not a setup question +### publish/sync merge generated files mechanically and carry non-overlapping uncommitted work through a rebase + +Every ingest touches `kb/index.md` and `kb/log.md`, so two ingests on two checkouts - or two +sessions - always "overlapped", in files that carry no decision. `reconcile` (behind `sync` and +both of `publish`'s reconciles) failed on all three shapes of that: an uncommitted ingest stopped +the fast-forward with exit 1; two committed ingests went to the rebase-review gate and, once +cleared, failed on the two appends at the end of the log; and any uncommitted file at all made +`git rebase` refuse, so a stranded commit plus a new ingest could not be published. + +- **Generated files never count as an overlap.** The rebase-review gate, its token and its diff + leave out `kb/index.md`, `kb/log.md`, `kb/provenance.md` and every `kb/**/INDEX.md`; an + overlapping *non*-generated file still goes to review. +- **Where they are what stops git, they are merged mechanically.** Uncommitted generated files + are set aside (the log's appended bytes kept, the rest discarded); rebase conflicts in them are + resolved step by step - the catalog and provenance take the side being rebased onto, the log + takes that side plus what the replayed commit appended. Afterwards the log gets the set-aside + entries back at its end and the catalog and `kb/provenance.md` are regenerated in-process, left + as an uncommitted change. `index rebuild` and `sources rebuild-index` lend their writing half + for that as `index_build.write_index` and `provenance_cmd.write_provenance_index`. +- **Uncommitted work no incoming commit touches rides through a rebase** (`git rebase + --autostash`) and comes back byte-identical. +- **Before the first write, the working tree is stored** as a commit under + `refs/wikitool/reconcile-backup`; a success drops it, a failure restores the working tree and + names the ref. +- **`publish`'s retry after a rejected push** commits the regenerated files on their own before + the second push, since staging is long past by then. +- **Unchanged where git manages on its own:** a reconcile that succeeded before runs the same git + command and regenerates nothing - `index rebuild` stamps today's date into `kb/index.md`, and a + rebuild there would leave every `sync` with a modified file. A log edited rather than appended + to, or an uncommitted change on a path the incoming commits change, stops exactly as before. +- **`is_generated` is narrowed to `kb/`**, as AGENTS.md invariant 1 already defines it: an + `INDEX.md` in a captured repository under `raw/` is source material, counted by the Mass-Update + Gate and never discarded. + +Drop-in: no command, flag, file format or state file changes, and the previous version ignores a +leftover backup ref. Cases that ended in exit 1 or 42 now go through. A `--confirm-rebase` token +issued before the update may read as stale afterwards, because the overlap it covers no longer +lists generated files - the gate then asks again, as for any stale token. `sync`'s and +`publish`'s records, `instructions/session-setup.md` and `instructions/gates.md` follow (Gitea +#180). + ### kb/CONVENTIONS.md.template: the guideline filter is a default, not a setup question The ยง Guidelines for other repositories section shipped with a `{...}` placeholder, and diff --git a/VERSION b/VERSION index fe74fa4..0ae4a30 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -8.0.0-beta.45 +8.0.0-beta.46 diff --git a/instructions/gates.md b/instructions/gates.md index 90e2578..ca12c4c 100644 --- a/instructions/gates.md +++ b/instructions/gates.md @@ -36,7 +36,8 @@ Read the exit code first - it says which of these applies: A `wikitool` command that exits **42** is not reporting an error. It is refusing to act until a human has *read its output*. Five gates use it today - the Mass-Update Gate (`publish`, on a change touching 10 or more counted files), the rebase-review gate (`sync` and `publish`, on -a rebase whose incoming commits touch a file this session is also changing), the +a rebase whose incoming commits touch a file this session is also changing, generated files +aside), the Publish-Remote Gate (`publish`, on a push to a target this checkout has not declared), the Upload Review Gate (`upload accept`, on a submission nobody has cleared yet), and the Guideline Push Gate (`export guidelines --push`, before a generated `GUIDELINES.md` goes into any captured @@ -54,7 +55,9 @@ For the rebase-review gate the substance is different: the commits arriving from the files they touch that this session is also touching, and a diff of those files. Read it - this is the check `sync`/`publish` cannot perform themselves, since a rebase between two commit ranges that touch disjoint files never reaches this gate at all (no content collision is -possible by construction, so it rebases automatically). Judge whether the incoming change +possible by construction, so it rebases automatically). Nor does an overlap only in the files +`wikitool` generates - the catalog, `kb/log.md`, `kb/provenance.md` - which carry no decision +and are merged and regenerated mechanically; the gate never lists them. Judge whether the incoming change conflicts logically with what you are about to publish, summarize *that judgment*, not just the diff, to the user, and only then re-run with the `--confirm-rebase ` the refusal prints. diff --git a/instructions/session-setup.md b/instructions/session-setup.md index 05d87e5..b608dce 100644 --- a/instructions/session-setup.md +++ b/instructions/session-setup.md @@ -85,7 +85,11 @@ than one machine or session writes to. Running `sync` first shrinks that window the session instead of discovering the drift only at the very end. `sync` fetches the remote and fast-forwards or rebases automatically when that is safe; it -never commits and never pushes. **Exit 42 (rebase-review)?** Same as any exit 42 - read the +never commits and never pushes. Files `wikitool` generates are never a reason to stop: when the +catalog or `kb/log.md` changed on both sides, `sync` keeps both sides' log entries and +regenerates the catalog and `kb/provenance.md`, which it leaves as an uncommitted change for the +next `publish`. Uncommitted work that the incoming commits do not touch stays where it is. +**Exit 42 (rebase-review)?** Same as any exit 42 - read the diff it prints, judge whether it conflicts with what you are about to do, summarize that to the user, then `tools/wikitool sync --confirm-rebase ` before continuing. See [gates.md](gates.md). diff --git a/tools/CONTRACT.md b/tools/CONTRACT.md index e4a81fe..7ac8140 100644 --- a/tools/CONTRACT.md +++ b/tools/CONTRACT.md @@ -1888,7 +1888,7 @@ Fetch `/` and bring the local branch up to date with it. - effect: write - idempotent: yes -- atomic: No - fetch, then at most one merge/rebase attempt, aborted cleanly on failure +- atomic: No - fetch, then at most one merge/rebase attempt, aborted cleanly on failure with the working tree restored; a success that merged generated files then rewrites the catalog - budget: counted - network: yes - gates: rebase-review @@ -1903,13 +1903,15 @@ Fetch `/` and bring the local branch up to date with it. - 0 success - 0 No remote configured - reported and skipped, not a failure - 0 The remote cannot be reached - reported and skipped, not a failure -- 1 The automatic rebase hit a real conflict (git failed); it is aborted cleanly -- 42 Rebase-review gate: `/` moved and both sides changed the same file; the output lists the upstream commits, the overlapping files and their diff +- 1 The automatic rebase hit a real conflict outside the generated files (git failed), or `kb/log.md` was edited rather than appended to; it is aborted cleanly and the working tree restored +- 1 An uncommitted change - or an untracked file - sits on a path the incoming commits also change; nothing was changed +- 42 Rebase-review gate: `/` moved and both sides changed the same file, generated files aside; the output lists the upstream commits, the overlapping files and their diff **ON FAILURE** -- The automatic rebase hit a real conflict (git failed); it is aborted cleanly -> Do not retry and do not force - resolve the conflict manually, then re-run -- Rebase-review gate: `/` moved and both sides changed the same file; the output lists the upstream commits, the overlapping files and their diff -> Show the user the command's full output verbatim and stop. Once they have approved it, run the re-run line the output prints, which carries `--confirm-rebase `. Without that token, or with a wrong, invented or superseded one, it exits 42 again with the current state +- The automatic rebase hit a real conflict outside the generated files (git failed), or `kb/log.md` was edited rather than appended to; it is aborted cleanly and the working tree restored -> Do not retry and do not force - resolve the conflict manually, then re-run +- An uncommitted change - or an untracked file - sits on a path the incoming commits also change; nothing was changed -> Show the user the file it names. Once that change is committed or moved out of the way, re-run once +- Rebase-review gate: `/` moved and both sides changed the same file, generated files aside; the output lists the upstream commits, the overlapping files and their diff -> Show the user the command's full output verbatim and stop. Once they have approved it, run the re-run line the output prints, which carries `--confirm-rebase `. Without that token, or with a wrong, invented or superseded one, it exits 42 again with the current state **NEVER** @@ -1919,8 +1921,11 @@ Fetch `/` and bring the local branch up to date with it. **NOTES** - Fetches `/`, then: fast-forwards when only the remote moved; rebases the local commits on top when both sides moved but touched disjoint files; exits 42 (rebase-review gate) when both sides touched the same file. +- Files `wikitool` generates (`kb/index.md`, `kb/log.md`, `kb/provenance.md`, every `kb/**/INDEX.md`) never count as an overlap and never reach the gate. Where they are what stops git - uncommitted ones in the way of a fast-forward or rebase, or a rebase conflict in them - they are merged mechanically: `kb/log.md` becomes the incoming log plus the entries only this side appended, and the catalog and `kb/provenance.md` are regenerated from the merged pages and left as an uncommitted change. A reconcile git can do on its own is unchanged and regenerates nothing. +- Uncommitted changes that no incoming commit touches no longer stop a rebase: they are carried through it (`git rebase --autostash`) and come back unchanged. +- Before anything is set aside, the working tree is stored as a commit under `refs/wikitool/reconcile-backup`. A successful reconcile drops it; a failed one restores the working tree and names the ref. - A refused call performs no rebase attempt and leaves the branch where it was. -- The `--confirm-rebase` token covers the exact upstream state and the set of files touched on both sides; either one moving makes it stale. +- The `--confirm-rebase` token covers the exact upstream state and the set of files touched on both sides, generated files excluded; either one moving makes it stale. - Makes no commit, no push, and no forced operation of any kind. - Three messages for a fetch that fails: no remote of that name, a remote that answers but has no such branch yet (a new, empty repository), and a remote that cannot be reached. All three exit 0 here; `publish` stops on the first and the last. - Run it once at the start of a writing session. @@ -1957,7 +1962,7 @@ Reconcile with `/`, then stage all changes, commit, and push. **EXIT STATUS** - 0 success -- 1 git failed - `git add`, `git commit`, `git push`, or the reconcile's automatic rebase +- 1 git failed - `git add`, `git commit`, `git push`, or the reconcile's automatic rebase (a conflict outside the generated files, an edited `kb/log.md`, or an uncommitted change on a path the incoming commits change) - 1 The push target (`--branch`) is not the checked-out branch, or HEAD is detached; the unborn branch of a fresh `git init` is not this case - 1 No `--no-push`, and the remote is not configured or cannot be reached; nothing was committed - 1 `--yes`/`-y` was passed - the flag does not exist and fails with an explicit error @@ -1968,7 +1973,7 @@ Reconcile with `/`, then stage all changes, commit, and push. **ON FAILURE** -- git failed - `git add`, `git commit`, `git push`, or the reconcile's automatic rebase -> Do not retry and do not force - report and ask the user. `publish` has already made its one retry of a rejected push itself, where a reconcile resolved the rejection +- git failed - `git add`, `git commit`, `git push`, or the reconcile's automatic rebase (a conflict outside the generated files, an edited `kb/log.md`, or an uncommitted change on a path the incoming commits change) -> Do not retry and do not force - report and ask the user. `publish` has already made its one retry of a rejected push itself, where a reconcile resolved the rejection - The push target (`--branch`) is not the checked-out branch, or HEAD is detached; the unborn branch of a fresh `git init` is not this case -> Check out the branch you mean to publish, or pass `--branch `, then retry once - No `--no-push`, and the remote is not configured or cannot be reached; nothing was committed -> Show the message to the user and ask whether to commit locally with `--no-push`. Never push by hand - the next `publish` that reaches the remote sends that commit - `--yes`/`-y` was passed - the flag does not exist and fails with an explicit error -> Drop it. The Mass-Update Gate is cleared only with `--confirm ` from the gate's own refusal output @@ -1988,12 +1993,13 @@ Reconcile with `/`, then stage all changes, commit, and push. - Order: branch check and Publish-Remote Gate, then the reconcile with `/`, then the Mass-Update Gate, then `git add -A`, commit and push. `--no-push` skips all but the Mass-Update Gate and the commit. - Without `--no-push`, a remote that is not configured or cannot be reached ends the call with exit 1 at the reconcile - before the gate, `git add` and the commit, and also on a clean tree. Nothing is committed, the index and the working tree are unchanged, and the message names `--no-push` as the way to a local commit. The next `publish` that reaches the remote pushes that commit along with whatever is new. A remote that answers but has no `` yet (a new, empty repository) is not this case: the first publish of an instance commits and pushes as before. - Reconcile: fetches `/`, fast-forwards when only the remote moved, rebases the local commits on top when both sides moved but touched disjoint files, and exits 42 (rebase-review gate) when both sides touched the same file. A refused reconcile performs no rebase attempt. The `--confirm-rebase` token covers the exact upstream state and the set of files touched on both sides. +- Generated files are never an overlap: where they are all that stops the reconcile, it merges them as `sync` does (both sides' log entries kept, the catalog and `kb/provenance.md` regenerated) and carries uncommitted changes no incoming commit touches through the rebase. The proactive reconcile runs before staging, so this publish commits the regenerated files; on the retry after a rejected push they get a commit of their own before the second push. - The push target must be the checked-out branch; this is checked before anything is staged. The unborn branch of a fresh `git init -b main` counts as checked out, so the first publish of a new instance works; a real detached HEAD is refused. - With nothing new to stage, a local commit the remote lacks is still pushed: one left behind by an earlier publish whose push failed, or every commit when the remote answers but does not have the branch yet (a new, empty remote repository). - A rejected push that finds the remote unreachable on its one retry reports the original push error. - A rejected push gets exactly one more reconcile-and-push; never more than one. - Mass-Update Gate: counts the files that would be committed, refuses with exit 42 at `--threshold` (default 10) or more, and prints a review report - a scale line (file count, total lines added/removed, status breakdown), attention notes where they apply (deletions by name, control-plane and harness-config touches, published pages, the largest single change, binaries), and every counted path grouped by area with its status and churn. The list is what the commit will hold: it is computed from a scratch copy of the index after `git add -A`, so a path that is staged as deleted and back in the working tree is not counted twice, and a rename counts as its old path deleted plus its new path added. The real index and the working tree are not touched, so a refused publish leaves both byte-identical. -- Never counted and never shown for approval, but committed like everything else: anything under `work/`, and the files `wikitool` generates itself (`kb/index.md`, `kb/log.md`, `kb/provenance.md`, every `INDEX.md`). The refusal line accounts for both, by reason. +- Never counted and never shown for approval, but committed like everything else: anything under `work/`, and the files `wikitool` generates itself (`kb/index.md`, `kb/log.md`, `kb/provenance.md`, every `INDEX.md` under `kb/`). The refusal line accounts for both, by reason. - The `--confirm` token covers each counted path, the blob id of its contents and the publish target: a different file list or edited contents need a new clearance. - Publish-Remote Gate: when the checkout carries `.wikitool-remotes.json` and the push URL of `--remote` is not listed in it, exits 42 before the reconcile fetches anything. The URL is read with `git remote get-url --push`, so a repointed remote does not pass on its name. An absent file means unrestricted; a malformed one is an error, not permission. - `--path` (repeatable) scopes the whole operation - gate count, staging and commit - to that subtree. diff --git a/tools/chemenu/commands/git_publish.py b/tools/chemenu/commands/git_publish.py index b17d5d6..47b72b5 100644 --- a/tools/chemenu/commands/git_publish.py +++ b/tools/chemenu/commands/git_publish.py @@ -12,7 +12,9 @@ same exit-42 idiom as the gate below - when it might. This is what keeps a rejected push from stranding the commit that made it: a previous `publish` whose push failed leaves a real, unpushed commit sitting on the branch, and the next `publish` now pushes it instead of reporting "Nothing to commit" -forever. +forever. Files `wikitool` generates never count as a collision: where they +are what stops git, the reconcile merges them mechanically and regenerates +the catalog (see "Reconciling around generated files" below). Also implements the Mass-Update Gate (kb/concepts/workflows/Mass-Update Gate.md): a push to origin/main is the one action in this system with a real, @@ -80,12 +82,14 @@ DEFAULT_MASS_UPDATE_THRESHOLD = 10 GATE_EXEMPT_PREFIXES = ("work/",) -def _run(args: list[str], input: Optional[str] = None) -> subprocess.CompletedProcess: - """Run a `git ...` argument list, starting git from its recorded path.""" +def _run(args: list[str], input: Optional[str] = None, + env: Optional[dict[str, str]] = None) -> subprocess.CompletedProcess: + """Run a `git ...` argument list, starting git from its recorded path. `env` is laid over + the inherited environment, not substituted for it.""" if args and args[0] == "git": args = [toolpaths.git(), *args[1:]] return subprocess.run(args, cwd=config.ROOT, capture_output=True, text=True, encoding="utf-8", - input=input) + input=input, env={**os.environ, **env} if env else None) # --- Publish-Remote Gate ----------------------------------------------------- @@ -352,12 +356,17 @@ def branch_mismatch_message(checked_out: Optional[str], branch: str) -> str: # them pushed ordinary single-source ingests over a threshold meant for mass # updates. Three consecutive ingests on 2026-08-31 each stopped at the gate with # 5-7 of their files generated; none of them was a mass update. +# +# Only under `kb/`, which is also what AGENTS.md invariant 1 names (`kb/**/INDEX.md`). An +# `INDEX.md` in a captured repository under `raw/` is source material that happens to share the +# name - and `reconcile` discards and regenerates what this predicate calls generated. GENERATED_SUFFIXES = ("/INDEX.md",) GENERATED_PATHS = ("kb/index.md", "kb/log.md", "kb/provenance.md") +GENERATED_LOG = "kb/log.md" def is_generated(path: str) -> bool: - return path in GENERATED_PATHS or path.endswith(GENERATED_SUFFIXES) + return path in GENERATED_PATHS or (path.startswith("kb/") and path.endswith(GENERATED_SUFFIXES)) def is_exempt(path: str, prefixes: tuple[str, ...] = GATE_EXEMPT_PREFIXES) -> bool: @@ -767,12 +776,276 @@ def rebase_review_token(remote: str, branch: str, local_before: str, remote_tip: return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:12] +# --- Reconciling around generated files --------------------------------------- +# +# Every ingest touches kb/index.md and kb/log.md, so two ingests on two checkouts always +# "overlap" - in files that carry no decision and can be recomputed from the result. Left to +# git, that overlap stopped every such reconcile: a fast-forward refused over the uncommitted +# catalog, a rebase went to review and then failed on the two appends at the end of the log, +# and `git rebase` refused outright over any uncommitted file at all. +# +# The path below takes the generated files out of git's way and puts the result back +# together mechanically: the log is the incoming version plus what only this side appended, +# and the catalog and kb/provenance.md are regenerated from the merged pages. It runs only +# where git would otherwise have failed, so a reconcile that succeeds today is unchanged - +# in particular it does not rebuild, because `index rebuild` stamps today's date into +# kb/index.md and every `sync` would then end with a modified file. +# +# Uncommitted work that is not generated is never discarded: it rides through a rebase in +# `--autostash` and comes back unchanged, which needs only that no incoming commit touches it. +# Before the first write, the whole working tree is stored under BACKUP_REF. + +BACKUP_REF = "refs/wikitool/reconcile-backup" + +# The identity of the backup commit, so taking one never depends on - or fails for want of - +# a configured user. +_BACKUP_IDENTITY = { + "GIT_AUTHOR_NAME": "wikitool", "GIT_AUTHOR_EMAIL": "wikitool@localhost", + "GIT_COMMITTER_NAME": "wikitool", "GIT_COMMITTER_EMAIL": "wikitool@localhost", +} + +# A rebase that stops on a conflict this module resolves is continued without an editor. +# GIT_EDITOR and not `-c core.editor`, which an exported GIT_EDITOR would override. +_NO_EDITOR = {"GIT_EDITOR": "true"} + + +def _git_bytes(*args: str) -> Optional[bytes]: + """stdout of `git ` as raw bytes, or None when git fails. Bytes, because a file's + content goes back to disk exactly as git holds it - text mode would fold line endings.""" + result = subprocess.run([toolpaths.git(), *args], cwd=config.ROOT, capture_output=True) + return result.stdout if result.returncode == 0 else None + + +def _dirty_paths() -> tuple[list[str], list[str]]: + """(tracked paths whose index or working copy differs from HEAD, untracked paths). Ignored + files are neither - nothing here ever touches one.""" + result = _run(["git", "status", "--porcelain=v1", "-z", "--untracked-files=all", "--no-renames"]) + tracked: list[str] = [] + untracked: list[str] = [] + for record in result.stdout.split("\0"): + if len(record) < 4: + continue + (untracked if record[:2] == "??" else tracked).append(record[3:]) + return tracked, untracked + + +def _read(path: str) -> Optional[bytes]: + target = config.ROOT / path + return target.read_bytes() if target.is_file() else None + + +def _log_suffix() -> Optional[bytes]: + """The bytes the working copy of kb/log.md has beyond its HEAD version - b"" when it has + none - or None when the working copy is not the HEAD version with something appended. + `log append` only ever appends, so None means a hand edit, and the log is then left to + git as before.""" + work = _read(GENERATED_LOG) + head = _git_bytes("show", f"HEAD:{GENERATED_LOG}") + if work is None: + return b"" if head is None else None + head = head or b"" + return work[len(head):] if work.startswith(head) else None + + +def _in_head(path: str) -> bool: + return _run(["git", "cat-file", "-e", f"HEAD:{path}"]).returncode == 0 + + +def _rebase_in_progress() -> bool: + return any((config.ROOT / _git_path(name)).exists() for name in ("rebase-merge", "rebase-apply")) + + +def _back_up_working_tree() -> Optional[str]: + """Store the working tree as it stands - tracked and untracked files, not ignored ones - + as a commit on top of HEAD under BACKUP_REF, and return its id; None if that failed. + Staged through a scratch copy of the index, the same way `collect_changes` stages, so the + real index is not touched.""" + index_path = config.ROOT / _git_path("index") + scratch = Path(tempfile.mkdtemp(prefix="wikitool-backup-")) + try: + temp_index = scratch / "index" + if index_path.is_file(): + shutil.copyfile(index_path, temp_index) + env = {"GIT_INDEX_FILE": str(temp_index)} + if _run(["git", "add", "-A"], env=env).returncode != 0: + return None + tree = _run(["git", "write-tree"], env=env) + finally: + shutil.rmtree(scratch, ignore_errors=True) + if tree.returncode != 0: + return None + commit = _run( + ["git", "commit-tree", tree.stdout.strip(), "-p", "HEAD", + "-m", "wikitool reconcile: the working tree before generated files were set aside"], + env=_BACKUP_IDENTITY, + ) + if commit.returncode != 0: + return None + commit_id = commit.stdout.strip() + if _run(["git", "update-ref", BACKUP_REF, commit_id]).returncode != 0: + return None + return commit_id + + +@dataclass +class _SetAside: + """The generated files taken out of git's way: each one's working-copy bytes before + (None where it did not exist), and the bytes only this side appended to kb/log.md.""" + originals: dict[str, Optional[bytes]] + log_suffix: bytes + + +def _set_aside(paths: list[str]) -> _SetAside: + """Return each generated path in `paths` to its HEAD state - or remove it, where HEAD + has none - after recording what was there.""" + aside = _SetAside( + originals={path: _read(path) for path in paths}, + log_suffix=(_log_suffix() or b"") if GENERATED_LOG in paths else b"", + ) + in_head = [path for path in paths if _in_head(path)] + if in_head: + _run(["git", "checkout", "HEAD", "--", *in_head]) + for path in paths: + if path not in in_head: + _run(["git", "rm", "-q", "--cached", "--ignore-unmatch", "--", path]) + (config.ROOT / path).unlink(missing_ok=True) + return aside + + +def _put_back(aside: _SetAside) -> None: + """Undo `_set_aside` byte for byte in the working tree.""" + for path, data in aside.originals.items(): + target = config.ROOT / path + if data is None: + target.unlink(missing_ok=True) + else: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + + +def _generated_to_set_aside(incoming: set[str]) -> tuple[Optional[list[str]], str]: + """The dirty generated paths, or (None, why) when uncommitted work stands in the way of + reconciling with `incoming` at all - a non-generated file both sides change, or a + kb/log.md that was edited rather than appended to. Reads only.""" + tracked, untracked = _dirty_paths() + for path in tracked: + if not is_generated(path) and path in incoming: + return None, f"{path} has uncommitted changes and the incoming commits change it too" + for path in untracked: + if not is_generated(path) and path in incoming: + return None, f"the untracked {path} would be overwritten by an incoming commit" + generated = sorted(path for path in tracked + untracked if is_generated(path)) + if GENERATED_LOG in generated and _log_suffix() is None: + return None, f"{GENERATED_LOG} has uncommitted changes that are not only appended entries" + return generated, "" + + +def _resolve_generated_conflicts() -> Optional[str]: + """Resolve the conflicts of a stopped rebase step when every one of them is in a generated + file; None on success, else why not. Stage 2 is the side being rebased onto, stage 3 the + commit being replayed: the catalog and provenance take stage 2 (they are regenerated + afterwards anyway), the log takes stage 2 plus what stage 3 appended to stage 1.""" + listed = _run(["git", "diff", "--name-only", "--diff-filter=U", "-z"]).stdout + unmerged = [path for path in listed.split("\0") if path] + if not unmerged: + return "the rebase stopped without a file conflict" + foreign = [path for path in unmerged if not is_generated(path)] + if foreign: + return f"conflict in {', '.join(foreign)}" + for path in unmerged: + ours = _git_bytes("show", f":2:{path}") + target = config.ROOT / path + if path == GENERATED_LOG: + base = _git_bytes("show", f":1:{path}") or b"" + theirs = _git_bytes("show", f":3:{path}") + if ours is None or theirs is None or not theirs.startswith(base): + return f"{path} was changed on one side by more than appended entries" + target.write_bytes(ours + theirs[len(base):]) + elif ours is None: + _run(["git", "rm", "-q", "--cached", "--", path]) + target.unlink(missing_ok=True) + continue + else: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(ours) + if _run(["git", "add", "--", path]).returncode != 0: + return f"git add {path} failed" + return None + + +def _rebase_resolving_generated(remote_ref: str) -> tuple[bool, str, bool]: + """`git rebase --autostash `, resolving each step whose conflicts are all in + generated files. Returns (succeeded, detail, resolved anything). Any other stop is aborted, + which also hands the autostash back.""" + result = _run(["git", "rebase", "--autostash", remote_ref], env=_NO_EDITOR) + resolved = False + while result.returncode != 0: + if not _rebase_in_progress(): + return False, result.stderr, resolved + reason = _resolve_generated_conflicts() + if reason: + _run(["git", "rebase", "--abort"]) + return False, f"{reason}\n{result.stderr}", resolved + resolved = True + result = _run(["git", "rebase", "--continue"], env=_NO_EDITOR) + return True, "", resolved + + +def regenerate_generated(log_suffix: bytes = b"") -> None: + """Append `log_suffix` to kb/log.md, then rewrite the catalog and kb/provenance.md from the + pages in the working tree - what `index rebuild` and `sources rebuild-index` write. The log + goes first: it is the one part that cannot be recomputed.""" + from chemenu.commands.index_build import write_index + from chemenu.commands.provenance_cmd import write_provenance_index + + if log_suffix: + with (config.ROOT / GENERATED_LOG).open("ab") as log: + log.write(log_suffix) + write_index(config.KB_DIR) + write_provenance_index(config.KB_DIR, config.RAW_DIR) + + +def _around_generated(generated: list[str], operation, regenerate: bool) -> tuple[bool, str, bool]: + """Back up the working tree, set `generated` aside, run `operation` - which returns + (succeeded, detail, resolved generated conflicts) - and then either regenerate or put + everything back. Returns (succeeded, detail, regenerated). + + Regenerates when something generated was set aside or resolved, or when `regenerate` says + both sides' commits touched a generated file - a textual merge of two catalogs can succeed + and still be wrong. The backup ref is dropped on success and kept, and named, on any + failure.""" + backup = _back_up_working_tree() + if backup is None: + return False, "could not back up the working tree first, so nothing was changed", False + aside = _set_aside(generated) + try: + succeeded, detail, resolved = operation() + except BaseException: + if _rebase_in_progress(): + _run(["git", "rebase", "--abort"]) + _put_back(aside) + raise + if not succeeded: + _put_back(aside) + return False, ( + f"{detail.rstrip()}\nThe working tree was restored; as it stood before, it is also " + f"kept at {BACKUP_REF} ({backup[:12]})." + ), False + regenerated = bool(generated) or resolved or regenerate + if regenerated: + regenerate_generated(aside.log_suffix) + _run(["git", "update-ref", "-d", BACKUP_REF]) + return True, "", regenerated + + @dataclass class ReconcileOutcome: """What happened when the local branch was brought up to date with the remote before a publish/sync. `status` is one of: "no-remote", "remote-lacks-branch", "unreachable", "up-to-date", "fast-forwarded", "local-ahead", "rebased", "needs-review", "conflict". - The first three are the ways the fetch can fail; `detail` then carries the remote's URL.""" + The first three are the ways the fetch can fail; `detail` then carries the remote's URL. + `regenerated` says the catalog and kb/provenance.md were rewritten afterwards and are an + uncommitted change now.""" status: str pulled_commits: list[str] = field(default_factory=list) overlap_files: list[str] = field(default_factory=list) @@ -780,6 +1053,7 @@ class ReconcileOutcome: token: str = "" was_reviewed: bool = False detail: str = "" + regenerated: bool = False def reconcile(remote: str, branch: str, confirm_rebase: Optional[str] = None) -> ReconcileOutcome: @@ -791,6 +1065,11 @@ def reconcile(remote: str, branch: str, confirm_rebase: Optional[str] = None) -> actually rewrites history: an overlap that has not been cleared performs no rebase attempt at all, so a refused call leaves the branch exactly where it was found. + Generated files (`is_generated`) are left out of the overlap: where they - or uncommitted + work that no incoming commit touches - are all that stops git, the reconcile goes ahead + around them (see "Reconciling around generated files" above) and leaves the regenerated + catalog as an uncommitted change. + Never commits, never pushes, never force-anything. Callers are `sync` (this is its whole job) and `publish` (proactively before staging, and once more if the eventual push is rejected - the real, narrow race this whole module exists to close).""" @@ -812,12 +1091,24 @@ def reconcile(remote: str, branch: str, confirm_rebase: Optional[str] = None) -> if state == "ff-possible": pulled = oneline_log(branch, remote_ref) result = _run(["git", "merge", "--ff-only", remote_ref]) - if result.returncode != 0: - # Only reachable if uncommitted local changes would be overwritten by the merge - - # git itself refuses and touches nothing, so this is a safe abort, not a half-done - # state. + if result.returncode == 0: + return ReconcileOutcome(status="fast-forwarded", pulled_commits=pulled) + # Only reachable if uncommitted local changes would be overwritten by the merge - + # git itself refuses and touches nothing, so this is a safe abort, not a half-done + # state. When the only such changes are generated files, they are set aside instead. + incoming = touched_files("HEAD", remote_ref) + generated, _why = _generated_to_set_aside(incoming) + if not generated or not incoming.intersection(generated): return ReconcileOutcome(status="conflict", detail=result.stderr) - return ReconcileOutcome(status="fast-forwarded", pulled_commits=pulled) + + def fast_forward() -> tuple[bool, str, bool]: + retry = _run(["git", "merge", "--ff-only", remote_ref]) + return retry.returncode == 0, retry.stderr, False + + succeeded, detail, regenerated = _around_generated(generated, fast_forward, False) + if not succeeded: + return ReconcileOutcome(status="conflict", detail=detail) + return ReconcileOutcome(status="fast-forwarded", pulled_commits=pulled, regenerated=regenerated) if state == "local-ahead": return ReconcileOutcome(status="local-ahead") @@ -830,7 +1121,10 @@ def reconcile(remote: str, branch: str, confirm_rebase: Optional[str] = None) -> return ReconcileOutcome(status="conflict", detail="no common history with remote") upstream_commits = oneline_log(base, remote_tip) - overlap = sorted(touched_files(base, local_before) & touched_files(base, remote_tip)) + incoming = touched_files(base, remote_tip) + both_sides = touched_files(base, local_before) & incoming + overlap = sorted(path for path in both_sides if not is_generated(path)) + generated_overlap = any(is_generated(path) for path in both_sides) was_reviewed = False token = "" @@ -844,13 +1138,34 @@ def reconcile(remote: str, branch: str, confirm_rebase: Optional[str] = None) -> ) was_reviewed = True - rebase_result = _run(["git", "rebase", remote_ref]) - if rebase_result.returncode != 0: - _run(["git", "rebase", "--abort"]) - return ReconcileOutcome(status="conflict", detail=rebase_result.stderr) + tracked, untracked = _dirty_paths() + in_the_way = tracked + [path for path in untracked if is_generated(path)] + if not in_the_way and not generated_overlap: + # Nothing generated on both sides and nothing uncommitted that `git rebase` refuses + # over: the plain rebase, exactly as it always ran. + rebase_result = _run(["git", "rebase", remote_ref]) + if rebase_result.returncode != 0: + _run(["git", "rebase", "--abort"]) + return ReconcileOutcome(status="conflict", detail=rebase_result.stderr) + return ReconcileOutcome( + status="rebased", pulled_commits=upstream_commits, + overlap_files=overlap, was_reviewed=was_reviewed, token=token, + ) + + generated, why = _generated_to_set_aside(incoming) + if generated is None: + return ReconcileOutcome( + status="conflict", + detail=f"Nothing was changed: {why}. Commit or set aside that change, then re-run.", + ) + succeeded, detail, regenerated = _around_generated( + generated, lambda: _rebase_resolving_generated(remote_ref), generated_overlap, + ) + if not succeeded: + return ReconcileOutcome(status="conflict", detail=detail) return ReconcileOutcome( status="rebased", pulled_commits=upstream_commits, - overlap_files=overlap, was_reviewed=was_reviewed, token=token, + overlap_files=overlap, was_reviewed=was_reviewed, token=token, regenerated=regenerated, ) @@ -920,13 +1235,26 @@ def rebase_review_message(outcome: ReconcileOutcome, remote: str, branch: str, c def _reconcile_summary(outcome: ReconcileOutcome, remote: str, branch: str) -> str: """One line for the outcomes that do not fail or gate - what to tell the caller, or "" for `conflict` and `needs-review`, which `apply_reconcile` raises on instead.""" + regenerated = ( + " Generated files were merged mechanically: kb/log.md keeps both sides' entries, and the " + "catalog and kb/provenance.md were regenerated - an uncommitted change now." + if outcome.regenerated else "" + ) if outcome.status == "fast-forwarded": - return f"Pulled {len(outcome.pulled_commits)} commit(s) from {remote}/{branch}." + return f"Pulled {len(outcome.pulled_commits)} commit(s) from {remote}/{branch}.{regenerated}" if outcome.status == "local-ahead": return f"Local branch is ahead of {remote}/{branch}; nothing to pull." if outcome.status == "rebased": - reviewed = " after review" if outcome.was_reviewed else " (no file overlap, automatic)" - return f"Rebased onto {len(outcome.pulled_commits)} new commit(s) from {remote}/{branch}{reviewed}." + if outcome.was_reviewed: + how = " after review" + elif outcome.regenerated: + how = " (no overlap outside generated files, automatic)" + else: + how = " (no file overlap, automatic)" + return ( + f"Rebased onto {len(outcome.pulled_commits)} new commit(s) from " + f"{remote}/{branch}{how}.{regenerated}" + ) if outcome.status == "up-to-date": return f"Already up to date with {remote}/{branch}." if outcome.status == "no-remote": @@ -1003,7 +1331,9 @@ def apply_reconcile(outcome: ReconcileOutcome, remote: str, branch: str, command properties=cli_contract.Properties( effect=cli_contract.Effect.WRITE, idempotent=cli_contract.Idempotent.YES, - atomic="No - fetch, then at most one merge/rebase attempt, aborted cleanly on failure", + atomic="No - fetch, then at most one merge/rebase attempt, aborted cleanly on failure " + "with the working tree restored; a success that merged generated files then rewrites " + "the catalog", budget=cli_contract.Budget.COUNTED, network=cli_contract.Network.YES, gates=("rebase-review",), @@ -1012,9 +1342,21 @@ def apply_reconcile(outcome: ReconcileOutcome, remote: str, branch: str, command "Fetches `/`, then: fast-forwards when only the remote moved; rebases " "the local commits on top when both sides moved but touched disjoint files; exits 42 " "(rebase-review gate) when both sides touched the same file.", + "Files `wikitool` generates (`kb/index.md`, `kb/log.md`, `kb/provenance.md`, every " + "`kb/**/INDEX.md`) never count as an overlap and never reach the gate. Where they are " + "what stops git - uncommitted ones in the way of a fast-forward or rebase, or a rebase " + "conflict in them - they are merged mechanically: `kb/log.md` becomes the incoming log " + "plus the entries only this side appended, and the catalog and `kb/provenance.md` are " + "regenerated from the merged pages and left as an uncommitted change. A reconcile git " + "can do on its own is unchanged and regenerates nothing.", + "Uncommitted changes that no incoming commit touches no longer stop a rebase: they are " + "carried through it (`git rebase --autostash`) and come back unchanged.", + "Before anything is set aside, the working tree is stored as a commit under " + "`refs/wikitool/reconcile-backup`. A successful reconcile drops it; a failed one " + "restores the working tree and names the ref.", "A refused call performs no rebase attempt and leaves the branch where it was.", "The `--confirm-rebase` token covers the exact upstream state and the set of files " - "touched on both sides; either one moving makes it stale.", + "touched on both sides, generated files excluded; either one moving makes it stale.", "Makes no commit, no push, and no forced operation of any kind.", "Three messages for a fetch that fails: no remote of that name, a remote that answers " "but has no such branch yet (a new, empty repository), and a remote that cannot be " @@ -1033,13 +1375,21 @@ def apply_reconcile(outcome: ReconcileOutcome, remote: str, branch: str, command code=0, ), cli_contract.Failure( - cause="The automatic rebase hit a real conflict (git failed); it is aborted cleanly", + cause="The automatic rebase hit a real conflict outside the generated files (git " + "failed), or `kb/log.md` was edited rather than appended to; it is aborted cleanly " + "and the working tree restored", reaction="Do not retry and do not force - resolve the conflict manually, then re-run", ), + cli_contract.Failure( + cause="An uncommitted change - or an untracked file - sits on a path the incoming " + "commits also change; nothing was changed", + reaction="Show the user the file it names. Once that change is committed or moved " + "out of the way, re-run once", + ), cli_contract.Failure( cause="Rebase-review gate: `/` moved and both sides changed the " - "same file; the output lists the upstream commits, the overlapping files and their " - "diff", + "same file, generated files aside; the output lists the upstream commits, the " + "overlapping files and their diff", reaction=cli_contract.token_gate_reaction("--confirm-rebase"), code=42, ), @@ -1116,6 +1466,24 @@ def commit_message(message: str, changed_files: list[str]) -> str: return candidate if _trailers(candidate) == trailers else plain +def commit_regenerated(remote: str, branch: str) -> None: + """Commit the generated files a reconcile just rewrote - and only those, whatever else is + staged - with a message of their own.""" + tracked, untracked = _dirty_paths() + paths = sorted(path for path in tracked + untracked if is_generated(path)) + if not paths: + return + add_result = _run(["git", "add", "-A", "--", *paths]) + if add_result.returncode != 0: + fail(f"git add failed:\n{add_result.stderr}") + message = commit_message( + f"chore: regenerate the catalog after reconciling with {remote}/{branch}", paths, + ) + commit_result = _run(["git", "commit", "-m", message, "--", *paths]) + if commit_result.returncode != 0: + fail(f"git commit failed:\n{commit_result.stderr}") + + @cli_contract.record(cli_contract.CommandRecord( path="publish", summary="Reconcile with `/`, then stage all changes, commit, and push.", @@ -1150,6 +1518,12 @@ def commit_message(message: str, changed_files: list[str]) -> str: "and exits 42 (rebase-review gate) when both sides touched the same file. A refused " "reconcile performs no rebase attempt. The `--confirm-rebase` token covers the exact " "upstream state and the set of files touched on both sides.", + "Generated files are never an overlap: where they are all that stops the reconcile, it " + "merges them as `sync` does (both sides' log entries kept, the catalog and " + "`kb/provenance.md` regenerated) and carries uncommitted changes no incoming commit " + "touches through the rebase. The proactive reconcile runs before staging, so this " + "publish commits the regenerated files; on the retry after a rejected push they get a " + "commit of their own before the second push.", "The push target must be the checked-out branch; this is checked before anything is " "staged. The unborn branch of a fresh `git init -b main` counts as checked out, so the " "first publish of a new instance works; a real detached HEAD is refused.", @@ -1171,8 +1545,8 @@ def commit_message(message: str, changed_files: list[str]) -> str: "refused publish leaves both byte-identical.", "Never counted and never shown for approval, but committed like everything else: " "anything under `work/`, and the files `wikitool` generates itself (`kb/index.md`, " - "`kb/log.md`, `kb/provenance.md`, every `INDEX.md`). The refusal line accounts for " - "both, by reason.", + "`kb/log.md`, `kb/provenance.md`, every `INDEX.md` under `kb/`). The refusal line " + "accounts for both, by reason.", "The `--confirm` token covers each counted path, the blob id of its contents and the " "publish target: a different file list or edited contents need a new clearance.", "Publish-Remote Gate: when the checkout carries `.wikitool-remotes.json` and the push " @@ -1199,7 +1573,8 @@ def commit_message(message: str, changed_files: list[str]) -> str: failures=( cli_contract.Failure( cause="git failed - `git add`, `git commit`, `git push`, or the reconcile's " - "automatic rebase", + "automatic rebase (a conflict outside the generated files, an edited `kb/log.md`, " + "or an uncommitted change on a path the incoming commits change)", reaction="Do not retry and do not force - report and ask the user. `publish` has " "already made its one retry of a rejected push itself, where a reconcile resolved " "the rejection", @@ -1443,6 +1818,10 @@ def publish_command( retry_summary = apply_reconcile(retry_outcome, remote, branch, "publish") if retry_summary: typer.echo(retry_summary) + if retry_outcome.regenerated: + # Staging is long past here, so the regenerated catalog would otherwise + # stay behind and the push would carry the stale one it was merged from. + commit_regenerated(remote, branch) push_result = _run(["git", "push", remote, branch]) elif retry_outcome.status in ("conflict", "needs-review"): apply_reconcile(retry_outcome, remote, branch, "publish") # raises diff --git a/tools/chemenu/commands/index_build.py b/tools/chemenu/commands/index_build.py index 41affd7..2af124d 100644 --- a/tools/chemenu/commands/index_build.py +++ b/tools/chemenu/commands/index_build.py @@ -210,6 +210,21 @@ def stale_shards(kb_dir: Path, plan: dict[Path, str]) -> list[Path]: return sorted(p for p in kb_dir.rglob(GENERATED_INDEX) if p not in plan) +def write_index(kb_dir: Path) -> tuple[dict[Path, str], list[Path]]: + """Write every catalog file and remove every stale shard; returns (plan, removed). + + The writing half of `index rebuild`, as a function so `reconcile` can regenerate the + catalog in-process after it has merged two sides' generated files.""" + plan = plan_index(kb_dir) + stale = stale_shards(kb_dir, plan) + for path, content in plan.items(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8", newline="\n") + for path in stale: + path.unlink() + return plan, stale + + def build_index(kb_dir: Path) -> str: """The root map. Kept as a named function because callers (and tests) ask for "the index" meaning the entry point, not the whole plan.""" @@ -276,24 +291,17 @@ def index_rebuild( "See `wikitool lint`'s Nested Pages finding." ) - plan = plan_index(config.KB_DIR) - stale = stale_shards(config.KB_DIR, plan) - if dry_run: + plan = plan_index(config.KB_DIR) for path in sorted(plan): typer.echo(f"--- {rel_path(path)}") # nl=False: the content already ends in a newline. typer.echo(plan[path], nl=False) - for path in stale: + for path in stale_shards(config.KB_DIR, plan): typer.echo(f"--- would remove stale shard: {rel_path(path)}") return - for path, content in plan.items(): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content, encoding="utf-8", newline="\n") - for path in stale: - path.unlink() - + plan, stale = write_index(config.KB_DIR) shards = len(plan) - 1 removed = f", removed {len(stale)} stale" if stale else "" success(f"Rebuilt {rel_path(config.INDEX_FILE)} and {shards} shard(s){removed}") diff --git a/tools/chemenu/commands/provenance_cmd.py b/tools/chemenu/commands/provenance_cmd.py index 569361c..19b1d10 100644 --- a/tools/chemenu/commands/provenance_cmd.py +++ b/tools/chemenu/commands/provenance_cmd.py @@ -231,6 +231,14 @@ def build_provenance_index(kb_dir: Path, raw_dir: Path) -> str: return "\n".join(lines) + "\n" +def write_provenance_index(kb_dir: Path, raw_dir: Path) -> Path: + """Write `kb/provenance.md` from scratch and return its path - the writing half of + `sources rebuild-index`, callable in-process by `reconcile`.""" + provenance_file = kb_dir / "provenance.md" + provenance_file.write_text(build_provenance_index(kb_dir, raw_dir), encoding="utf-8", newline="\n") + return provenance_file + + @app.command("rebuild-index") @cli_contract.record(cli_contract.CommandRecord( path="sources rebuild-index", @@ -267,12 +275,10 @@ def rebuild_index( dry_run: bool = typer.Option(False, "--dry-run", help="Print the result instead of writing kb/provenance.md"), ): """Regenerate the `kb/provenance.md` reverse index.""" - content = build_provenance_index(config.KB_DIR, config.RAW_DIR) - provenance_file = config.KB_DIR / "provenance.md" if dry_run: # nl=False so the preview is byte-identical to the file that would be # written; see the same note in index_build.py. - typer.echo(content, nl=False) + typer.echo(build_provenance_index(config.KB_DIR, config.RAW_DIR), nl=False) return - provenance_file.write_text(content, encoding="utf-8", newline="\n") + provenance_file = write_provenance_index(config.KB_DIR, config.RAW_DIR) success(f"Rebuilt {rel_path(provenance_file)}") diff --git a/tools/chemenu/tests/test_git_publish.py b/tools/chemenu/tests/test_git_publish.py index 0001ed2..3156827 100644 --- a/tools/chemenu/tests/test_git_publish.py +++ b/tools/chemenu/tests/test_git_publish.py @@ -1,4 +1,5 @@ import json +import shutil import subprocess import pytest @@ -367,8 +368,8 @@ def _publish(**overrides): """Call `publish_command` directly. Every parameter must be given a real value: bypassing Typer's CLI parsing means an omitted argument keeps its `typer.Option(...)` sentinel instead of the value it wraps.""" - kwargs = dict(message="change", push=True, confirm=None, yes=False, threshold=10, - remote="origin", branch="main", path=None) + kwargs = dict(message="change", push=True, confirm=None, confirm_rebase=None, yes=False, + threshold=10, remote="origin", branch="main", path=None) kwargs.update(overrides) publish_command(**kwargs) @@ -515,6 +516,14 @@ def test_generated_files_are_recognised_wherever_they_sit(): assert not is_generated("kb/concepts/protocols/Modbus.md") +def test_an_index_md_outside_kb_is_not_generated(): + """A captured repository under raw/ may carry its own INDEX.md. It is source material, and + `reconcile` discards and regenerates whatever this predicate calls generated.""" + assert not is_generated("raw/repos/project/INDEX.md") + assert not is_generated("docs/INDEX.md") + assert counted_files(["raw/repos/project/INDEX.md"]) == ["raw/repos/project/INDEX.md"] + + def test_paths_land_in_the_group_a_reviewer_expects(): assert group_of("kb/concepts/X.md")[0] == "Published knowledge" assert group_of("AGENTS.md")[0] == "Agent control plane" @@ -1497,3 +1506,395 @@ def test_a_mass_update_token_does_not_depend_on_the_message(repo): assert _git( repo, "log", "-1", "--format=%(trailers:key=Co-Authored-By,valueonly)" ).stdout.strip() == "A " + + +# --- Reconciling around generated files: two ingests on two checkouts ----------------------- +# +# Every ingest touches kb/index.md and kb/log.md. These tests build a real catalog with the +# shipped rebuild functions, so "the catalog is current" is checked against what `index +# rebuild`/`sources rebuild-index` actually write, not against a hand-made stand-in. + +from contextlib import contextmanager + +from chemenu.commands.log_append import format_log_entry, parse_log_entries +from chemenu.frontmatter_io import write_page +from chemenu.tests.conftest import use_shipped_type_specs + +GENERATED_FILES = ("kb/index.md", "kb/log.md", "kb/provenance.md", "kb/concepts/INDEX.md") + + +@contextmanager +def _at(root): + """Point `config.ROOT` at another checkout for the duration - the writer clone builds its + catalog with the same functions, against its own tree.""" + previous = config.ROOT + config.ROOT = root + try: + yield + finally: + config.ROOT = previous + + +def _write_concept(root, title, text="A concept.\n"): + write_page( + root / f"kb/concepts/protocols/{title}.md", + {"type": "types/concept.md", "concept_type": "protocol", "tags": [], + "created": "2026-10-06", "modified": "2026-10-06", "related": [], "sources": []}, + f"\n# {title}\n\n## Definition\n\n{text}", + ) + + +def _ingest(root, title, text="A concept.\n"): + """What an ingest leaves in the working tree: a page, the regenerated catalog and + provenance, and one appended log entry.""" + _write_concept(root, title, text) + with _at(root): + git_publish.regenerate_generated() + with (root / "kb/log.md").open("a", encoding="utf-8", newline="\n") as log: + log.write("\n" + format_log_entry("ingest", title, today="2026-10-06")) + + +def _commit_all(root, message): + _git(root, "add", "-A") + _git(root, "commit", "-m", message) + + +@pytest.fixture +def wiki_repo(repo, monkeypatch): + """`repo` with a real, pushed kb/: one collection, one page, its catalog and provenance + built by the shipped functions, and a log.""" + use_shipped_type_specs(monkeypatch) + (repo / "kb/concepts").mkdir(parents=True) + (repo / "kb/concepts/COLLECTION.md").write_text( + "---\nprofile: concepts\nrequired_by_stack: false\noutbound:\n any: [see-also]\n---\n\n" + "# kb/concepts/ - Collection Contract\n", + encoding="utf-8", + ) + (repo / "raw/notes").mkdir(parents=True) + (repo / "raw/notes/a.md").write_text("raw\n", encoding="utf-8") + (repo / "kb/log.md").write_text("# Wiki Log\n", encoding="utf-8") + _ingest(repo, "Base") + _commit_all(repo, "wiki") + _git(repo, "push", "origin", "main") + return repo + + +def _writer_ingest(writer, title): + _ingest(writer, title) + _commit_all(writer, f"writer ingest: {title}") + _git(writer, "push", "origin", "main") + + +def _tree(root): + """Every file of the working tree outside .git, as bytes.""" + return { + path.relative_to(root).as_posix(): path.read_bytes() + for path in root.rglob("*") + if path.is_file() and ".git" not in path.relative_to(root).parts + } + + +def _non_generated(tree): + return {path: data for path, data in tree.items() if not is_generated(path)} + + +def _assert_kept(before, root): + """Every non-generated file `before` holds is still there, byte for byte. Files the + incoming commits add are new, not changed.""" + after = _tree(root) + assert {path: after.get(path) for path in _non_generated(before)} == _non_generated(before) + + +def _check_tree(root): + """A fresh clone of the remote tip, as a place to compare its generated files against a + rebuild of that same tip.""" + check = root.parent / "check" + shutil.rmtree(check, ignore_errors=True) + subprocess.run(["git", "clone", str(_remote_for(root)), str(check)], check=True, + capture_output=True) + return check + + +def _assert_generated_current(root): + """The generated files at `root` are byte for byte what a rebuild of `root` writes today.""" + before = {path: (root / path).read_bytes() for path in GENERATED_FILES if path != "kb/log.md"} + with _at(root): + git_publish.regenerate_generated() + after = {path: (root / path).read_bytes() for path in before} + assert before == after + + +def _logged_titles(root): + return [title for _date, _op, title in parse_log_entries( + (root / "kb/log.md").read_text(encoding="utf-8"))] + + +def test_case_1_an_uncommitted_ingest_publishes_over_an_ingest_that_landed_meanwhile(wiki_repo): + writer = _clone_writer(wiki_repo) + _writer_ingest(writer, "Writer") + _ingest(wiki_repo, "Local") + + _publish(message="ingest: Local") # must not raise - no gate, no failure + + check = _check_tree(wiki_repo) + assert (check / "kb/concepts/protocols/Writer.md").is_file() + assert (check / "kb/concepts/protocols/Local.md").is_file() + assert _logged_titles(check) == ["Base", "Writer", "Local"] + _assert_generated_current(check) + assert _status(wiki_repo) == "" + + +def test_case_1_on_sync_leaves_the_regenerated_catalog_uncommitted(wiki_repo): + writer = _clone_writer(wiki_repo) + _writer_ingest(writer, "Writer") + _ingest(wiki_repo, "Local") + page = (wiki_repo / "kb/concepts/protocols/Local.md").read_bytes() + + outcome = reconcile("origin", "main", None) + + assert (outcome.status, outcome.regenerated) == ("fast-forwarded", True) + assert (wiki_repo / "kb/concepts/protocols/Local.md").read_bytes() == page + assert _logged_titles(wiki_repo) == ["Base", "Writer", "Local"] + _assert_generated_current(wiki_repo) + assert "kb/index.md" in _status(wiki_repo) + assert subprocess.run(["git", "show-ref", "--verify", "-q", git_publish.BACKUP_REF], + cwd=wiki_repo).returncode != 0 # dropped on success + + +def test_case_2_two_committed_ingests_rebase_without_the_review_gate(wiki_repo): + writer = _clone_writer(wiki_repo) + _writer_ingest(writer, "Writer") + _ingest(wiki_repo, "Local") + _commit_all(wiki_repo, "local ingest, push failed") # the stranded commit + + _publish(message="retry") # must not raise + + check = _check_tree(wiki_repo) + assert _logged_titles(check) == ["Base", "Writer", "Local"] + _assert_generated_current(check) + assert _status(wiki_repo) == "" + + +def test_case_2_on_sync_ends_with_exit_0_and_both_sides_in_the_log(wiki_repo): + writer = _clone_writer(wiki_repo) + _writer_ingest(writer, "Writer") + _ingest(wiki_repo, "Local") + _commit_all(wiki_repo, "local ingest") + + _sync() # must not raise + + assert _logged_titles(wiki_repo) == ["Base", "Writer", "Local"] + _assert_generated_current(wiki_repo) + assert "writer ingest: Writer" in _git(wiki_repo, "log", "--oneline").stdout + + +def test_case_2_still_gates_on_an_overlapping_page_and_names_no_generated_file(wiki_repo): + writer = _clone_writer(wiki_repo) + _ingest(writer, "Base", text="Writer's definition.\n") + _commit_all(writer, "writer edits Base") + _git(writer, "push", "origin", "main") + _ingest(wiki_repo, "Base", text="Local definition.\n") + _commit_all(wiki_repo, "local edits Base") + + outcome = reconcile("origin", "main", None) + + assert outcome.status == "needs-review" + assert outcome.overlap_files == ["kb/concepts/protocols/Base.md"] + assert "kb/log.md" not in outcome.overlap_diff and "kb/index.md" not in outcome.overlap_diff + + +def test_case_2_in_the_retry_path_commits_the_regenerated_catalog_before_the_push( + wiki_repo, monkeypatch +): + writer = _clone_writer(wiki_repo) + raced = {"done": False} + real_run = git_publish._run + + def racy_run(args, **kwargs): + if args[:2] == ["git", "push"] and not raced["done"]: + raced["done"] = True + _writer_ingest(writer, "Writer") + return real_run(args, **kwargs) + + monkeypatch.setattr(git_publish, "_run", racy_run) + _ingest(wiki_repo, "Local") + + _publish(message="ingest: Local") # must not raise + + assert raced["done"] + check = _check_tree(wiki_repo) + assert _logged_titles(check) == ["Base", "Writer", "Local"] + _assert_generated_current(check) + assert "chore: regenerate the catalog" in _git(check, "log", "-1", "--format=%s").stdout + assert _status(wiki_repo) == "" + + +def _stranded_ingest_plus_uncommitted_ingest(wiki_repo): + writer = _clone_writer(wiki_repo) + _writer_ingest(writer, "Writer") + _ingest(wiki_repo, "Stranded") + _commit_all(wiki_repo, "stranded ingest") + _ingest(wiki_repo, "Uncommitted") + (wiki_repo / "README.md").write_text("init\nedited, not committed\n", encoding="utf-8") + + +def test_case_3_publish_carries_uncommitted_work_over_a_stranded_commit(wiki_repo): + _stranded_ingest_plus_uncommitted_ingest(wiki_repo) + + _publish(message="ingest: Uncommitted") # must not raise + + check = _check_tree(wiki_repo) + assert (check / "kb/concepts/protocols/Uncommitted.md").is_file() + assert (check / "README.md").read_text(encoding="utf-8") == "init\nedited, not committed\n" + assert _logged_titles(check) == ["Base", "Writer", "Stranded", "Uncommitted"] + _assert_generated_current(check) + + +def test_case_3_sync_leaves_the_uncommitted_pages_exactly_as_they_were(wiki_repo): + _stranded_ingest_plus_uncommitted_ingest(wiki_repo) + before = _tree(wiki_repo) + + _sync() # must not raise + + _assert_kept(before, wiki_repo) + status = _status(wiki_repo) + assert "README.md" in status and "Uncommitted.md" in status # still uncommitted + assert _logged_titles(wiki_repo) == ["Base", "Writer", "Stranded", "Uncommitted"] + _assert_generated_current(wiki_repo) + + +def test_case_3_without_generated_files_rebases_and_rebuilds_nothing(repo, monkeypatch): + """A stranded commit and an unrelated uncommitted edit: `git rebase` used to refuse over + the edit. Nothing generated is involved, so nothing is regenerated.""" + monkeypatch.setattr(git_publish, "regenerate_generated", + lambda *a, **k: pytest.fail("nothing generated was involved")) + writer = _clone_writer(repo) + _push_from_writer(writer, "from-writer.md", "writer\n") + (repo / "local.md").write_text("local\n", encoding="utf-8") + _commit_all(repo, "stranded") + (repo / "README.md").write_text("uncommitted\n", encoding="utf-8") + + outcome = reconcile("origin", "main", None) + + assert (outcome.status, outcome.regenerated) == ("rebased", False) + assert (repo / "README.md").read_text(encoding="utf-8") == "uncommitted\n" + assert (repo / "from-writer.md").is_file() + + +@pytest.mark.parametrize("how", ["raises", "fails"]) +def test_uncommitted_work_survives_an_abort_in_the_middle_of_the_rebase(wiki_repo, monkeypatch, how): + """The invariant: whatever happens mid-rebase, every non-generated file in the working tree + is byte-identical afterwards, and the branch has not moved. A failure also restores the + generated files and names the backup ref.""" + _stranded_ingest_plus_uncommitted_ingest(wiki_repo) + before_tree = _tree(wiki_repo) + before_head = _git(wiki_repo, "rev-parse", "HEAD").stdout + + def forced(*_args): + if how == "raises": + raise RuntimeError("forced abort mid-rebase") + return "forced failure" + + monkeypatch.setattr(git_publish, "_resolve_generated_conflicts", forced) + + if how == "raises": + with pytest.raises(RuntimeError): + reconcile("origin", "main", None) + else: + outcome = reconcile("origin", "main", None) + assert outcome.status == "conflict" + assert git_publish.BACKUP_REF in outcome.detail + assert _tree(wiki_repo) == before_tree + + assert _non_generated(_tree(wiki_repo)) == _non_generated(before_tree) + assert _git(wiki_repo, "rev-parse", "HEAD").stdout == before_head + assert not git_publish._rebase_in_progress() + assert _git(wiki_repo, "stash", "list").stdout == "" + # The backup holds the working tree as it stood, uncommitted page included. + assert "Uncommitted" in _git( + wiki_repo, "show", f"{git_publish.BACKUP_REF}:kb/concepts/protocols/Uncommitted.md" + ).stdout + + +def _assert_untouched(repo, run): + tree, head = _tree(repo), _git(repo, "rev-parse", "HEAD").stdout + with pytest.raises(typer.Exit) as excinfo: + run() + assert excinfo.value.exit_code == 1 + assert _tree(repo) == tree + assert _git(repo, "rev-parse", "HEAD").stdout == head + + +def test_a_hand_edited_log_is_left_to_git_as_before(wiki_repo): + writer = _clone_writer(wiki_repo) + _writer_ingest(writer, "Writer") + _ingest(wiki_repo, "Local") + log = wiki_repo / "kb/log.md" + log.write_text(log.read_text(encoding="utf-8").replace("# Wiki Log", "# Edited Log"), + encoding="utf-8") + + _assert_untouched(wiki_repo, _sync) + + +def test_a_log_rewritten_in_a_local_commit_aborts_the_rebase_as_before(wiki_repo): + writer = _clone_writer(wiki_repo) + _writer_ingest(writer, "Writer") + _ingest(wiki_repo, "Local") + log = wiki_repo / "kb/log.md" + log.write_text(log.read_text(encoding="utf-8").replace("# Wiki Log", "# Edited Log"), + encoding="utf-8") + _commit_all(wiki_repo, "local ingest with a rewritten log") + + _assert_untouched(wiki_repo, _sync) + + +def test_an_uncommitted_change_to_a_file_the_incoming_commits_change_stops_as_before(wiki_repo): + writer = _clone_writer(wiki_repo) + _ingest(writer, "Base", text="Writer's definition.\n") + _commit_all(writer, "writer edits Base") + _git(writer, "push", "origin", "main") + _ingest(wiki_repo, "Stranded") + _commit_all(wiki_repo, "stranded") + _write_concept(wiki_repo, "Base", "Uncommitted local definition.\n") + + _assert_untouched(wiki_repo, _sync) + + +def test_an_untracked_file_on_an_incoming_path_stops_as_before(wiki_repo): + writer = _clone_writer(wiki_repo) + _writer_ingest(writer, "Writer") + _ingest(wiki_repo, "Stranded") + _commit_all(wiki_repo, "stranded") + _write_concept(wiki_repo, "Writer", "Same title, written here and never committed.\n") + + _assert_untouched(wiki_repo, _sync) + + +def test_a_plain_fast_forward_regenerates_nothing(wiki_repo, monkeypatch): + """None of the three cases: the result is what it always was, and in particular no rebuild + runs - it would stamp today's date into kb/index.md and leave every sync dirty.""" + writer = _clone_writer(wiki_repo) + _writer_ingest(writer, "Writer") + monkeypatch.setattr(git_publish, "regenerate_generated", + lambda *a, **k: pytest.fail("no rebuild without a collision")) + + outcome = reconcile("origin", "main", None) + + assert (outcome.status, outcome.regenerated) == ("fast-forwarded", False) + assert _status(wiki_repo) == "" + + +def test_an_index_md_under_raw_still_goes_to_review_when_both_sides_change_it(repo): + (repo / "raw/repos/project").mkdir(parents=True) + (repo / "raw/repos/project/INDEX.md").write_text("original\n", encoding="utf-8") + _commit_all(repo, "capture") + _git(repo, "push", "origin", "main") + writer = _clone_writer(repo) + _push_from_writer(writer, "raw/repos/project/INDEX.md", "writer\n") + (repo / "raw/repos/project/INDEX.md").write_text("local\n", encoding="utf-8") + _commit_all(repo, "local") + + outcome = reconcile("origin", "main", None) + + assert outcome.status == "needs-review" + assert outcome.overlap_files == ["raw/repos/project/INDEX.md"]