fix: upstream merge - preserve gitignored local data, refuse a merge git never opened (4.5.0-beta.3, #30)

Files changed:
- CHANGES.md
- VERSION
- docs/ownership-and-templates.md
- instructions/gates.md
- instructions/private-instance.md
- tools/CONTRACT.md
- tools/chemenu/commands/upstream_cmd.py
- tools/chemenu/tests/test_upstream_cmd.py

tools/chemenu/commands/upstream_cmd.py

@@ -22,7 +22,6 @@ same way.
 """
 from __future__ import annotations
 
-import shutil
 from pathlib import Path
 from typing import Optional
 
@@ -110,6 +109,52 @@ def _stack_paths_changed(since: str, until: str) -> list[str]:
 # --- upstream merge ---------------------------------------------------------
 
 
+def _prune_empty_dirs(stage: str) -> None:
+    """Remove directories left empty under `stage` after tracked files were
+    deleted. git tracks no directories, so an emptied one is invisible to
+    `git status` and would otherwise linger in the working tree as litter -
+    an empty `kb/<area>/` that only ever existed in the upstream's corpus.
+    Never touches a directory that still holds anything, ignored files
+    included."""
+    stage_dir = config.ROOT / stage
+    if not stage_dir.is_dir():
+        return
+    for path in sorted(stage_dir.rglob("*"), key=lambda p: len(p.parts), reverse=True):
+        if path.is_dir() and not any(path.iterdir()):
+            path.rmdir()
+
+
+def _restore_stage_to_local(stage: str, tracked_paths: set[str]) -> None:
+    """Force one content stage back to the local (HEAD) side, whatever the
+    merge did to it.
+
+    Deletes **only what git tracks on either side** - never the stage
+    directory wholesale. That distinction is the whole point of this function:
+    `reports/` is gitignored except its contract (see .gitignore), so a
+    content stage's working tree legitimately holds local data that is not in
+    any tree and not recomputable - the telemetry traces `eval score` reads,
+    saved eval reports, past lint reports. A blanket `rm -rf` of the stage
+    takes all of it out as collateral for a merge that was never about it.
+
+    Handles a stage that exists only in MERGE_HEAD too (the upstream
+    introduced it): what the merge wrote is removed, and there is simply
+    nothing to check out from HEAD afterwards.
+    """
+    prefix = f"{stage}/"
+    stage_paths = [p for p in tracked_paths if p.startswith(prefix)]
+    if not stage_paths:
+        return
+
+    _run(["git", "rm", "-rq", "--cached", "--ignore-unmatch", stage])
+    for relative in stage_paths:
+        target = config.ROOT / relative
+        if target.is_file() or target.is_symlink():
+            target.unlink()
+    _prune_empty_dirs(stage)
+    if _tree_has_path("HEAD", stage):
+        _run(["git", "checkout", "HEAD", "--", stage])
+
+
 def _remote_gate_warning() -> None:
     if git_publish.read_allowed_push_urls() is not None:
         return
@@ -167,16 +212,28 @@ def _postcheck_failure_message(leaks: list[str], before: str) -> str:
 
 
 def _merge_success_message(
-    updated: list[str], deleted: list[str], remote: str, branch: str
+    changed: list[str], deleted: list[str], remote: str, branch: str
 ) -> str:
-    lines = [f"Merged {remote}/{branch}. Content stages ({', '.join(ownership.CONTENT_STAGES)}) are unchanged."]
-    if updated:
-        lines.append(f"Stack paths updated ({len(updated)}):")
-        lines += [f"  - {p}" for p in updated]
-    if deleted:
-        lines.append(f"Stack paths removed, following the upstream ({len(deleted)}):")
-        lines += [f"  - {p}" for p in deleted]
-    if not updated and not deleted:
+    """What the merge actually did, measured against the pre-merge commit
+    rather than against what was restored.
+
+    `changed` is the real diff - restoring every stack-owned path from
+    MERGE_HEAD touches each of them whether or not the upstream moved any, so
+    reporting the restore list would claim seven updates for a merge that
+    changed one file, and a reader who checks would find the report wrong.
+    """
+    deleted_set = set(deleted)
+    lines = [
+        f"Merged {remote}/{branch}. Content stages "
+        f"({', '.join(ownership.CONTENT_STAGES)}) are unchanged."
+    ]
+    if changed:
+        lines.append(f"Stack paths changed ({len(changed)}):")
+        lines += [
+            f"  - {p}" + (" (deleted, following the upstream)" if p in deleted_set else "")
+            for p in changed
+        ]
+    else:
         lines.append("No stack-owned path changed.")
     return "\n".join(lines)
 
@@ -224,24 +281,36 @@ def merge_command(
         success(f"Already up to date with {remote_ref}.")
         return
 
-    _run(["git", "merge", "--no-commit", "--no-ff", remote_ref])
-
-    for stage in ownership.CONTENT_STAGES:
-        if not _tree_has_path("HEAD", stage):
-            continue
-        _run(["git", "rm", "-rq", "--cached", "--ignore-unmatch", stage])
-        stage_dir = config.ROOT / stage
-        if stage_dir.exists():
-            shutil.rmtree(stage_dir)
-        _run(["git", "checkout", "HEAD", "--", stage])
+    # The exit code is deliberately not the test - conflicts under the content
+    # stages are expected here and are exactly what the next steps undo. What
+    # *is* load-bearing is that a merge actually opened: without MERGE_HEAD,
+    # `_tree_paths("MERGE_HEAD")` is empty, and every stack-owned path in HEAD
+    # would then read as "the upstream deleted it" and be removed. A merge git
+    # refused to start (unrelated histories, an ignored file in the way) must
+    # therefore stop here, with the tree untouched.
+    merge_result = _run(["git", "merge", "--no-commit", "--no-ff", remote_ref])
+    if not _merge_in_progress():
+        fail(
+            f"`git merge --no-commit --no-ff {remote_ref}` did not open a merge, so there is "
+            f"nothing to scope - the working tree is unchanged:\n"
+            f"{merge_result.stdout}{merge_result.stderr}"
+        )
+        return
 
     merge_head_paths = _tree_paths("MERGE_HEAD")
     head_paths = _tree_paths("HEAD")
+    tracked_paths = merge_head_paths | head_paths
+
+    for stage in ownership.CONTENT_STAGES:
+        _restore_stage_to_local(stage, tracked_paths)
+
     stack_paths = sorted(
         p for p in (merge_head_paths | head_paths) if ownership.is_stack_owned(p)
     )
 
-    updated: list[str] = []
+    # Only the deletions are recorded: what was *restored* is every stack-owned
+    # path in MERGE_HEAD, which is not the same question as what changed - the
+    # success message asks git for that instead.
     deleted: list[str] = []
     for relative in stack_paths:
         if relative in merge_head_paths:
@@ -252,7 +321,6 @@ def merge_command(
                     f"in MERGE_HEAD's own tree:\n{checkout.stderr}\nThe merge is left open."
                 )
                 return
-            updated.append(relative)
         else:
             _run(["git", "rm", "-q", "--cached", "--ignore-unmatch", relative])
             target = config.ROOT / relative
@@ -275,7 +343,9 @@ def merge_command(
         fail(_postcheck_failure_message(leaks, before))
         return
 
-    success(_merge_success_message(updated, deleted, remote, branch))
+    success(
+        _merge_success_message(_stack_paths_changed(before, "HEAD"), deleted, remote, branch)
+    )
 
 
 # --- upstream verify ---------------------------------------------------------