feat: wikitool upstream merge/verify - code procedure for taking a stack update (4.5.0-beta.1, #30)

Files changed:
- AGENTS.md
- CHANGES.md
- VERSION
- instructions/gates.md
- instructions/private-instance.md
- tools/CONTRACT.md
- tools/README.md
- tools/chemenu/cli.py
- tools/chemenu/commands/dist_cmd.py
- tools/chemenu/commands/run_budget.py
- tools/chemenu/commands/upstream_cmd.py
- tools/chemenu/ownership.py
- tools/chemenu/tests/test_upstream_cmd.py

tools/chemenu/commands/dist_cmd.py

@@ -39,7 +39,7 @@ from typing import Callable, NamedTuple, Optional, Union
 
 import typer
 
-from chemenu import config, conventions, kb_collections, kb_state, version as version_mod
+from chemenu import config, conventions, kb_collections, kb_state, ownership, version as version_mod
 from chemenu.commands._util import fail, rel_path, success, today_iso
 
 app = typer.Typer(help="Build a distributable copy of the wiki machinery.")
@@ -132,8 +132,15 @@ INSTRUCTIONS_EXCLUDE_DIRS = {"dev"}
 RAW_SUBDIRS = ("articles", "documents", "notes", "assets")
 
 # Stage contracts that are not collections and carry no pages: copied as a
-# single file each, nothing else from their directory.
-CONTRACT_ONLY_STAGES = ("raw/CONTRACT.md", "reports/CONTRACT.md", "work/CONTRACT.md")
+# single file each, nothing else from their directory. `kb/` is excluded here
+# - it is a content stage too, but it has collections underneath it, so its
+# contract is handled by `build_plan` alongside them rather than as a bare
+# stage copy. Derived from `ownership.CONTENT_STAGES` rather than listed
+# again, so the set this loop copies and the set `upstream merge` restores
+# cannot name a different stage without one of them failing its own test.
+CONTRACT_ONLY_STAGES = tuple(
+    f"{stage}/CONTRACT.md" for stage in ownership.CONTENT_STAGES if stage != "kb"
+)
 
 # Single tracked files copied out of an otherwise-untouched, partially-ignored
 # directory. `.claude/` holds the harness's own session-tracing config
@@ -439,19 +446,20 @@ def build_plan(origin: Optional[Origin] = None) -> dict[str, PlannedFile]:
 # appears in INSTALL.md and version.py, so such a scan would either whitelist
 # the very string it is looking for or cry wolf on every export.
 #
-# `COLLECTION.md` and `CONVENTIONS.md` are deliberately *not* on the allowed
-# list any more. Both bind, and both are the instance's to write, so they cross
-# the boundary as `.template` and are adopted by a rename - a plan carrying the
+# `COLLECTION.md` and `CONVENTIONS.md` are deliberately *not* allowed through
+# any more. Both bind, and both are the instance's to write, so they cross the
+# boundary as `.template` and are adopted by a rename - a plan carrying the
 # filled name would hand a new instance this one's authoring conventions as
 # though they were the stack's.
+#
+# What counts as machinery under kb/ or raw/ is no longer a second list here:
+# it is `ownership.is_stack_owned`, the same predicate `upstream merge` and
+# `upstream verify` restore/check against. Only the export-only stubs
+# (`ownership.EXPORT_STUB_NAMES`) are allowed here without also being
+# stack-owned - a merge keeps the *local* copy of those, while export writes a
+# fresh one regardless of either side, so the two callers genuinely disagree
+# about them and each keeps its own allowance for that one case.
 _CONTENT_PREFIXES = ("kb/", "raw/")
-_CONTENT_ALLOWED_NAMES = (
-    "CONTRACT.md",
-    f"{kb_collections.CONTRACT_NAME}.template",
-    conventions.CONVENTIONS_TEMPLATE,
-    "log.md",
-    ".gitkeep",
-)
 _INSTANCE_OWNED_KB_FILES = (kb_collections.CONTRACT_NAME, conventions.CONVENTIONS_FILENAME)
 
 
@@ -473,7 +481,11 @@ def find_leaks(plan: dict[str, PlannedFile]) -> list[str]:
             leaks.append(f"{relative} (this instance's page type-spec; ship the .template)")
         elif relative.startswith("instructions/dev/"):
             leaks.append(f"{relative} (stack-development only)")
-        elif relative.startswith(_CONTENT_PREFIXES) and name not in _CONTENT_ALLOWED_NAMES:
+        elif (
+            relative.startswith(_CONTENT_PREFIXES)
+            and not ownership.is_stack_owned(relative)
+            and not ownership.is_export_stub(name)
+        ):
             leaks.append(f"{relative} (wiki content, not machinery)")
     return leaks
 

tools/chemenu/commands/run_budget.py

@@ -101,6 +101,13 @@ SKIP_COMMAND_PATHS = {
     ("migrate", "list"),
     ("migrate", "status"),
     ("migrate", "verify"),
+    # `upstream verify` only reads two git revisions and reports what changed -
+    # the same argument as `migrate verify`: a check that costs budget is one
+    # an agent starts skipping. `upstream merge` stays counted: it mutates the
+    # branch and can leave an open merge behind on refusal, so it belongs on
+    # the non-idempotent list (AGENTS.md's tool error contract) rather than
+    # the exempt one.
+    ("upstream", "verify"),
 }
 
 # Commands exempt regardless of their first argument, because that argument is

tools/chemenu/commands/upstream_cmd.py

@@ -0,0 +1,317 @@
+"""`wikitool upstream` - take a stack update from a public upstream into a
+private instance's `main` without letting the upstream's own content (a demo
+corpus, a workshop run) ride along.
+
+`git merge upstream/main` on its own treats a moved corpus dangerously
+asymmetrically: a page the instance deleted and the upstream edited reports as
+a conflict, a page the upstream *added* stages silently, and a page both sides
+deleted is the only harmless case. `instructions/private-instance.md`'s prose
+procedure closes that, by holding the merge open, forcing the content stages
+(`ownership.CONTENT_STAGES`) back to the local side, and then restoring only
+the paths `ownership.is_stack_owned` recognises as machinery. `upstream merge`
+is that procedure in code, so the path set it acts on cannot drift from the
+one `dist_cmd.py` ships - both read `chemenu.ownership` - and so a conflict in
+the machinery layers, or a machinery file the upstream deleted, gets an
+explained stop instead of a silently wrong commit.
+
+`upstream verify` is the other half: given two revisions, did anything change
+under a content stage except through a stack-owned path? It shares
+`_content_leaks` with the postcheck `upstream merge` runs on itself, so a
+hand-resolved merge or a future `dist upgrade` (Gitea #7) can be checked the
+same way.
+"""
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from chemenu import config, ownership
+from chemenu.commands import git_publish
+from chemenu.commands._util import console, fail, success
+
+app = typer.Typer(help="Take a stack update from a public upstream, machinery only.")
+
+
+def _run(args: list[str]):
+    import subprocess
+
+    return subprocess.run(args, cwd=config.ROOT, capture_output=True, text=True)
+
+
+def _rev_parse(rev: str) -> Optional[str]:
+    result = _run(["git", "rev-parse", "--verify", "-q", rev])
+    return result.stdout.strip() if result.returncode == 0 else None
+
+
+def _git_dir() -> Optional[Path]:
+    result = _run(["git", "rev-parse", "--git-dir"])
+    if result.returncode != 0:
+        return None
+    path = Path(result.stdout.strip())
+    return path if path.is_absolute() else config.ROOT / path
+
+
+def _working_tree_dirty() -> bool:
+    result = _run(["git", "status", "--porcelain"])
+    return bool(result.stdout.strip())
+
+
+def _merge_in_progress() -> bool:
+    git_dir = _git_dir()
+    return git_dir is not None and (git_dir / "MERGE_HEAD").exists()
+
+
+def _remote_resolves(remote: str) -> bool:
+    return _run(["git", "remote", "get-url", remote]).returncode == 0
+
+
+def _is_ancestor(ancestor: str, of: str) -> bool:
+    return _run(["git", "merge-base", "--is-ancestor", ancestor, of]).returncode == 0
+
+
+def _tree_has_path(rev: str, path: str) -> bool:
+    return _run(["git", "rev-parse", "--verify", "-q", f"{rev}:{path}"]).returncode == 0
+
+
+def _tree_paths(rev: str) -> set[str]:
+    result = _run(["git", "ls-tree", "-r", "--name-only", "-z", rev])
+    if result.returncode != 0:
+        return set()
+    return {p for p in result.stdout.split("\0") if p}
+
+
+def _content_leaks(since: str, until: str) -> list[str]:
+    """Paths under a content stage that changed between `since` and `until`
+    through something other than a stack-owned path. Shared by `upstream
+    merge`'s own postcheck and `upstream verify`, so the two cannot disagree
+    about what a clean update looks like."""
+    result = _run(["git", "diff", "--name-only", "-z", since, until, "--", *ownership.CONTENT_STAGES])
+    if result.returncode != 0:
+        fail(
+            f"`git diff {since} {until}` failed - is {since} a revision in this repository?\n"
+            f"{result.stderr}"
+        )
+        return []
+    changed = [p for p in result.stdout.split("\0") if p]
+    return sorted(p for p in changed if not ownership.is_stack_owned(p))
+
+
+def _stack_paths_changed(since: str, until: str) -> list[str]:
+    """The subset of the same diff that *is* a stack-owned path - the paths
+    that legitimately moved, for the success message."""
+    result = _run(["git", "diff", "--name-only", "-z", since, until, "--", *ownership.CONTENT_STAGES])
+    changed = [p for p in result.stdout.split("\0") if p]
+    return sorted(p for p in changed if ownership.is_stack_owned(p))
+
+
+# --- upstream merge ---------------------------------------------------------
+
+
+def _remote_gate_warning() -> None:
+    if git_publish.read_allowed_push_urls() is not None:
+        return
+    console.print(
+        "[bold yellow]WARN[/bold yellow] No .wikitool-remotes.json in this checkout - the "
+        "Publish-Remote Gate is unarmed, so a future `publish` to the wrong remote would not "
+        "be caught. `upstream merge` never pushes and proceeds regardless, but a checkout that "
+        "takes stack updates from a public upstream should arm the gate before its next publish "
+        "- see instructions/private-instance.md step 4."
+    )
+
+
+def _precondition_failure(remote: str) -> Optional[str]:
+    if _working_tree_dirty():
+        return (
+            "Working tree is not clean (`git status --porcelain` printed something). "
+            "`upstream merge` refuses to start on a dirty tree so a refusal never has to "
+            "guess which changes were already there. Commit or stash first."
+        )
+    if _merge_in_progress():
+        return (
+            "A merge is already in progress (.git/MERGE_HEAD exists). Resolve or abort it "
+            "(`git merge --abort`) before running `upstream merge`."
+        )
+    if not _remote_resolves(remote):
+        return f"Remote '{remote}' does not resolve (`git remote get-url {remote}` failed)."
+    return None
+
+
+def _unresolved_conflict_message(unresolved: list[str], remote: str, branch: str) -> str:
+    listed = "\n".join(f"  - {p}" for p in unresolved)
+    return (
+        f"A real conflict remains in the machinery layers after restoring the content stages "
+        f"and the stack-owned paths from {remote}/{branch}:\n{listed}\n\n"
+        "The merge is left open, uncommitted - nothing was written to the branch. Per "
+        "instructions/private-instance.md's decision points: this means the checkout changed "
+        "the stack locally, which private instances do not do. Take the upstream side for "
+        "these paths (`git checkout --theirs -- <path>` then `git add`) and re-file the local "
+        "change as an issue against the public repo, or resolve deliberately and "
+        "`git commit --no-edit` yourself. `git merge --abort` gives up the merge entirely."
+    )
+
+
+def _postcheck_failure_message(leaks: list[str], before: str) -> str:
+    listed = "\n".join(f"  - {p}" for p in leaks)
+    return (
+        f"The merge commit exists (content stages are not what they were before this ran), "
+        f"but it changed content outside of a stack-owned path:\n{listed}\n\n"
+        f"This was NOT rolled back - the state belongs in front of you, not behind an automatic "
+        f"repair the command applies to itself. Compare against the pre-merge commit ({before}) "
+        "and decide by hand whether to revert the merge commit, cherry-pick around it, or fix "
+        "forward. This is a bug in `upstream merge` or in `ownership.is_stack_owned` if it "
+        "reproduces - please report it rather than working around it silently."
+    )
+
+
+def _merge_success_message(
+    updated: list[str], deleted: list[str], remote: str, branch: str
+) -> str:
+    lines = [f"Merged {remote}/{branch}. Content stages ({', '.join(ownership.CONTENT_STAGES)}) are unchanged."]
+    if updated:
+        lines.append(f"Stack paths updated ({len(updated)}):")
+        lines += [f"  - {p}" for p in updated]
+    if deleted:
+        lines.append(f"Stack paths removed, following the upstream ({len(deleted)}):")
+        lines += [f"  - {p}" for p in deleted]
+    if not updated and not deleted:
+        lines.append("No stack-owned path changed.")
+    return "\n".join(lines)
+
+
+@app.command("merge")
+def merge_command(
+    remote: str = typer.Option("upstream", "--remote", help="Remote to merge from"),
+    branch: str = typer.Option("main", "--branch", help="Branch to merge"),
+    no_fetch: bool = typer.Option(
+        False, "--no-fetch", help="Skip `git fetch <remote>` - use whatever is already fetched"
+    ),
+):
+    """Merge `<remote>/<branch>` into the current branch, machinery only:
+    every path under a content stage (kb/, raw/, work/, reports/) is forced
+    back to the local side except a stack-owned path (`<stage>/CONTRACT.md`,
+    or anything ending `.template` under a content stage), which is taken
+    from the upstream - including a deletion, if the upstream removed one. A
+    real conflict elsewhere (tools/, types/, instructions/) leaves the merge
+    open and unresolved rather than guessing. Not idempotent: it can leave an
+    open merge behind on refusal. See instructions/private-instance.md."""
+    problem = _precondition_failure(remote)
+    if problem:
+        fail(problem)
+        return
+
+    _remote_gate_warning()
+
+    before = _rev_parse("HEAD")
+    if before is None:
+        fail("HEAD does not resolve - is this a git repository with at least one commit?")
+        return
+
+    if not no_fetch:
+        fetch_result = _run(["git", "fetch", remote, branch])
+        if fetch_result.returncode != 0:
+            fail(f"`git fetch {remote} {branch}` failed:\n{fetch_result.stderr}")
+            return
+
+    remote_ref = f"{remote}/{branch}"
+    if _rev_parse(remote_ref) is None:
+        fail(f"'{remote_ref}' does not resolve - fetch it first, or check --remote/--branch.")
+        return
+
+    if _is_ancestor(remote_ref, "HEAD"):
+        success(f"Already up to date with {remote_ref}.")
+        return
+
+    _run(["git", "merge", "--no-commit", "--no-ff", remote_ref])
+
+    for stage in ownership.CONTENT_STAGES:
+        if not _tree_has_path("HEAD", stage):
+            continue
+        _run(["git", "rm", "-rq", "--cached", "--ignore-unmatch", stage])
+        stage_dir = config.ROOT / stage
+        if stage_dir.exists():
+            shutil.rmtree(stage_dir)
+        _run(["git", "checkout", "HEAD", "--", stage])
+
+    merge_head_paths = _tree_paths("MERGE_HEAD")
+    head_paths = _tree_paths("HEAD")
+    stack_paths = sorted(
+        p for p in (merge_head_paths | head_paths) if ownership.is_stack_owned(p)
+    )
+
+    updated: list[str] = []
+    deleted: list[str] = []
+    for relative in stack_paths:
+        if relative in merge_head_paths:
+            checkout = _run(["git", "checkout", "MERGE_HEAD", "--", relative])
+            if checkout.returncode != 0:
+                fail(
+                    f"`git checkout MERGE_HEAD -- {relative}` failed even though it is listed "
+                    f"in MERGE_HEAD's own tree:\n{checkout.stderr}\nThe merge is left open."
+                )
+                return
+            updated.append(relative)
+        else:
+            _run(["git", "rm", "-q", "--cached", "--ignore-unmatch", relative])
+            target = config.ROOT / relative
+            if target.exists():
+                target.unlink()
+            deleted.append(relative)
+
+    unresolved = [p for p in _run(["git", "diff", "--name-only", "--diff-filter=U"]).stdout.splitlines() if p]
+    if unresolved:
+        fail(_unresolved_conflict_message(unresolved, remote, branch))
+        return
+
+    commit_result = _run(["git", "commit", "--no-edit"])
+    if commit_result.returncode != 0:
+        fail(f"`git commit --no-edit` failed:\n{commit_result.stderr}")
+        return
+
+    leaks = _content_leaks(before, "HEAD")
+    if leaks:
+        fail(_postcheck_failure_message(leaks, before))
+        return
+
+    success(_merge_success_message(updated, deleted, remote, branch))
+
+
+# --- upstream verify ---------------------------------------------------------
+
+
+def _verify_failure_message(leaks: list[str], since: str, until: str) -> str:
+    listed = "\n".join(f"  - {p}" for p in leaks)
+    return (
+        f"Content under a content stage (kb/, raw/, work/, reports/) changed between {since} "
+        f"and {until} through a path that is not stack-owned:\n{listed}\n\n"
+        "That is upstream content (or an equivalent local change) that reached this range "
+        "outside of a stack-owned path - inspect it before trusting this range as machinery-only."
+    )
+
+
+def _verify_success_message(stack_moved: list[str], since: str, until: str) -> str:
+    if not stack_moved:
+        return f"No content changed between {since} and {until} under kb/, raw/, work/, reports/."
+    listed = "\n".join(f"  - {p}" for p in stack_moved)
+    return (
+        f"Clean: only stack-owned paths changed under kb/, raw/, work/, reports/ between "
+        f"{since} and {until}:\n{listed}"
+    )
+
+
+@app.command("verify")
+def verify_command(
+    since: str = typer.Option(..., "--since", help="Git revision to compare from"),
+    until: str = typer.Option("HEAD", "--until", help="Git revision to compare to"),
+):
+    """Check that nothing under a content stage changed between --since and
+    --until except through a stack-owned path. Read-only, and exempt from the
+    Iteration Budget Gate - the same treatment `migrate verify` gets, for the
+    same reason: a check an agent has to ration is a check that gets skipped."""
+    leaks = _content_leaks(since, until)
+    if leaks:
+        fail(_verify_failure_message(leaks, since, until))
+        return
+    success(_verify_success_message(_stack_paths_changed(since, until), since, until))