fix: upstream merge - preserve gitignored local data, refuse a merge git never opened (4.5.0-beta.3, #30)

Files changed:
- CHANGES.md
- VERSION
- docs/ownership-and-templates.md
- instructions/gates.md
- instructions/private-instance.md
- tools/CONTRACT.md
- tools/chemenu/commands/upstream_cmd.py
- tools/chemenu/tests/test_upstream_cmd.py

tools/chemenu/commands/upstream_cmd.py

@@ -22,7 +22,6 @@ same way.
 """
 from __future__ import annotations
 
-import shutil
 from pathlib import Path
 from typing import Optional
 
@@ -110,6 +109,52 @@ def _stack_paths_changed(since: str, until: str) -> list[str]:
 # --- upstream merge ---------------------------------------------------------
 
 
+def _prune_empty_dirs(stage: str) -> None:
+    """Remove directories left empty under `stage` after tracked files were
+    deleted. git tracks no directories, so an emptied one is invisible to
+    `git status` and would otherwise linger in the working tree as litter -
+    an empty `kb/<area>/` that only ever existed in the upstream's corpus.
+    Never touches a directory that still holds anything, ignored files
+    included."""
+    stage_dir = config.ROOT / stage
+    if not stage_dir.is_dir():
+        return
+    for path in sorted(stage_dir.rglob("*"), key=lambda p: len(p.parts), reverse=True):
+        if path.is_dir() and not any(path.iterdir()):
+            path.rmdir()
+
+
+def _restore_stage_to_local(stage: str, tracked_paths: set[str]) -> None:
+    """Force one content stage back to the local (HEAD) side, whatever the
+    merge did to it.
+
+    Deletes **only what git tracks on either side** - never the stage
+    directory wholesale. That distinction is the whole point of this function:
+    `reports/` is gitignored except its contract (see .gitignore), so a
+    content stage's working tree legitimately holds local data that is not in
+    any tree and not recomputable - the telemetry traces `eval score` reads,
+    saved eval reports, past lint reports. A blanket `rm -rf` of the stage
+    takes all of it out as collateral for a merge that was never about it.
+
+    Handles a stage that exists only in MERGE_HEAD too (the upstream
+    introduced it): what the merge wrote is removed, and there is simply
+    nothing to check out from HEAD afterwards.
+    """
+    prefix = f"{stage}/"
+    stage_paths = [p for p in tracked_paths if p.startswith(prefix)]
+    if not stage_paths:
+        return
+
+    _run(["git", "rm", "-rq", "--cached", "--ignore-unmatch", stage])
+    for relative in stage_paths:
+        target = config.ROOT / relative
+        if target.is_file() or target.is_symlink():
+            target.unlink()
+    _prune_empty_dirs(stage)
+    if _tree_has_path("HEAD", stage):
+        _run(["git", "checkout", "HEAD", "--", stage])
+
+
 def _remote_gate_warning() -> None:
     if git_publish.read_allowed_push_urls() is not None:
         return
@@ -167,16 +212,28 @@ def _postcheck_failure_message(leaks: list[str], before: str) -> str:
 
 
 def _merge_success_message(
-    updated: list[str], deleted: list[str], remote: str, branch: str
+    changed: list[str], deleted: list[str], remote: str, branch: str
 ) -> str:
-    lines = [f"Merged {remote}/{branch}. Content stages ({', '.join(ownership.CONTENT_STAGES)}) are unchanged."]
-    if updated:
-        lines.append(f"Stack paths updated ({len(updated)}):")
-        lines += [f"  - {p}" for p in updated]
-    if deleted:
-        lines.append(f"Stack paths removed, following the upstream ({len(deleted)}):")
-        lines += [f"  - {p}" for p in deleted]
-    if not updated and not deleted:
+    """What the merge actually did, measured against the pre-merge commit
+    rather than against what was restored.
+
+    `changed` is the real diff - restoring every stack-owned path from
+    MERGE_HEAD touches each of them whether or not the upstream moved any, so
+    reporting the restore list would claim seven updates for a merge that
+    changed one file, and a reader who checks would find the report wrong.
+    """
+    deleted_set = set(deleted)
+    lines = [
+        f"Merged {remote}/{branch}. Content stages "
+        f"({', '.join(ownership.CONTENT_STAGES)}) are unchanged."
+    ]
+    if changed:
+        lines.append(f"Stack paths changed ({len(changed)}):")
+        lines += [
+            f"  - {p}" + (" (deleted, following the upstream)" if p in deleted_set else "")
+            for p in changed
+        ]
+    else:
         lines.append("No stack-owned path changed.")
     return "\n".join(lines)
 
@@ -224,24 +281,36 @@ def merge_command(
         success(f"Already up to date with {remote_ref}.")
         return
 
-    _run(["git", "merge", "--no-commit", "--no-ff", remote_ref])
-
-    for stage in ownership.CONTENT_STAGES:
-        if not _tree_has_path("HEAD", stage):
-            continue
-        _run(["git", "rm", "-rq", "--cached", "--ignore-unmatch", stage])
-        stage_dir = config.ROOT / stage
-        if stage_dir.exists():
-            shutil.rmtree(stage_dir)
-        _run(["git", "checkout", "HEAD", "--", stage])
+    # The exit code is deliberately not the test - conflicts under the content
+    # stages are expected here and are exactly what the next steps undo. What
+    # *is* load-bearing is that a merge actually opened: without MERGE_HEAD,
+    # `_tree_paths("MERGE_HEAD")` is empty, and every stack-owned path in HEAD
+    # would then read as "the upstream deleted it" and be removed. A merge git
+    # refused to start (unrelated histories, an ignored file in the way) must
+    # therefore stop here, with the tree untouched.
+    merge_result = _run(["git", "merge", "--no-commit", "--no-ff", remote_ref])
+    if not _merge_in_progress():
+        fail(
+            f"`git merge --no-commit --no-ff {remote_ref}` did not open a merge, so there is "
+            f"nothing to scope - the working tree is unchanged:\n"
+            f"{merge_result.stdout}{merge_result.stderr}"
+        )
+        return
 
     merge_head_paths = _tree_paths("MERGE_HEAD")
     head_paths = _tree_paths("HEAD")
+    tracked_paths = merge_head_paths | head_paths
+
+    for stage in ownership.CONTENT_STAGES:
+        _restore_stage_to_local(stage, tracked_paths)
+
     stack_paths = sorted(
         p for p in (merge_head_paths | head_paths) if ownership.is_stack_owned(p)
     )
 
-    updated: list[str] = []
+    # Only the deletions are recorded: what was *restored* is every stack-owned
+    # path in MERGE_HEAD, which is not the same question as what changed - the
+    # success message asks git for that instead.
     deleted: list[str] = []
     for relative in stack_paths:
         if relative in merge_head_paths:
@@ -252,7 +321,6 @@ def merge_command(
                     f"in MERGE_HEAD's own tree:\n{checkout.stderr}\nThe merge is left open."
                 )
                 return
-            updated.append(relative)
         else:
             _run(["git", "rm", "-q", "--cached", "--ignore-unmatch", relative])
             target = config.ROOT / relative
@@ -275,7 +343,9 @@ def merge_command(
         fail(_postcheck_failure_message(leaks, before))
         return
 
-    success(_merge_success_message(updated, deleted, remote, branch))
+    success(
+        _merge_success_message(_stack_paths_changed(before, "HEAD"), deleted, remote, branch)
+    )
 
 
 # --- upstream verify ---------------------------------------------------------

tools/chemenu/tests/test_upstream_cmd.py

@@ -55,7 +55,16 @@ def two_repos(tmp_path, monkeypatch):
     # fixture during a test would show up as an untracked file and trip the
     # dirty-working-tree precondition for a reason that has nothing to do with
     # what that test is checking.
-    _write(seed, ".gitignore", f"{config.PUBLISH_REMOTES_FILENAME}\n")
+    # Mirrors the real .gitignore in the two ways that matter here:
+    # `.wikitool-remotes.json` is per-checkout (dropping one in during a test
+    # must not read as a dirty tree), and `reports/` is derived output that is
+    # ignored except for its contract - which is what makes a content stage
+    # able to hold local, non-recomputable data a merge must not touch.
+    _write(
+        seed,
+        ".gitignore",
+        f"/{config.PUBLISH_REMOTES_FILENAME}\n/reports/*\n!/reports/CONTRACT.md\n",
+    )
     _write(seed, "kb/CONTRACT.md", "stack kb contract v1\n")
     _write(seed, "kb/CONVENTIONS.md.template", "template v1\n")
     _write(seed, "kb/CONVENTIONS.md", "instance conventions v1\n")
@@ -208,6 +217,47 @@ def test_open_workshop_run_files_do_not_land(two_repos):
     assert not (repo / "work/some-run").exists()
 
 
+def test_merge_keeps_ignored_local_data_under_a_content_stage(two_repos):
+    """`reports/` is gitignored except its contract, so a content stage's
+    working tree holds local data that is in no git tree and is not
+    recomputable - the telemetry traces `eval score` reads, saved eval
+    reports, past lint reports. Forcing the stage back to the local side must
+    not take those out as collateral: this instance had 497 trace directories
+    under reports/telemetry/ when the first version of this command wiped the
+    stage wholesale."""
+    upstream, repo = two_repos
+    _write(repo, "reports/telemetry/session-a/trace.jsonl", '{"event": "local"}\n')
+    _write(repo, "reports/Lint Report 2026-09-04.md", "a local lint report\n")
+    assert _git(repo, "status", "--porcelain").stdout == ""  # ignored, so the tree is clean
+
+    _write(upstream, "kb/CONTRACT.md", "stack kb contract v2\n")
+    _commit(upstream, "upstream changes kb/CONTRACT.md")
+
+    _merge()
+
+    assert (repo / "reports/telemetry/session-a/trace.jsonl").read_text(encoding="utf-8") == (
+        '{"event": "local"}\n'
+    )
+    assert (repo / "reports/Lint Report 2026-09-04.md").exists()
+    assert (repo / "reports/CONTRACT.md").read_text(encoding="utf-8") == "reports contract v1\n"
+
+
+def test_upstream_content_under_a_stage_absent_from_head_does_not_land(two_repos):
+    """The stage guard must not rest on the local side happening to track
+    something under that stage: an instance whose `work/` holds no tracked
+    file at all must still not receive the upstream's open run."""
+    upstream, repo = two_repos
+    _git(repo, "rm", "-q", "work/CONTRACT.md")
+    _commit(repo, "instance has nothing tracked under work/")
+
+    _write(upstream, "work/some-run/README.md", "an in-progress workshop run\n")
+    _commit(upstream, "upstream ships an open work/ run")
+
+    _merge()
+
+    assert not (repo / "work/some-run").exists()
+
+
 def test_one_upstream_commit_mixing_every_case_at_once(two_repos):
     """The acceptance test from the issue: a single upstream commit that edits
     a page the instance deleted, adds a new page, deletes an untouched page,
@@ -255,6 +305,51 @@ def test_real_conflict_in_tools_leaves_the_merge_open(two_repos):
     assert (repo / ".git" / "MERGE_HEAD").exists()
 
 
+def test_a_merge_git_refuses_to_open_deletes_nothing(two_repos, tmp_path):
+    """The failure mode with the worst blast radius if it is not guarded:
+    without MERGE_HEAD, every stack-owned path in HEAD reads as "the upstream
+    deleted it", and the restore loop would remove kb/CONTRACT.md,
+    raw/CONTRACT.md and every template. A merge git refuses to start must stop
+    before that, with the tree untouched."""
+    upstream, repo = two_repos
+    unrelated = tmp_path / "unrelated"
+    unrelated.mkdir()
+    _git(unrelated, "init", "-b", "main")
+    _git(unrelated, "config", "user.name", "Unrelated")
+    _git(unrelated, "config", "user.email", "unrelated@example.com")
+    _write(unrelated, "somefile.md", "no shared history with the instance\n")
+    _commit(unrelated, "unrelated root commit")
+
+    _git(repo, "remote", "set-url", "upstream", str(unrelated))
+    before = _git(repo, "rev-parse", "HEAD").stdout.strip()
+
+    with pytest.raises(typer.Exit) as excinfo:
+        _merge()
+    assert excinfo.value.exit_code == 1
+
+    assert _git(repo, "rev-parse", "HEAD").stdout.strip() == before
+    assert (repo / "kb/CONTRACT.md").read_text(encoding="utf-8") == "stack kb contract v1\n"
+    assert (repo / "raw/CONTRACT.md").exists()
+    assert (repo / "kb/CONVENTIONS.md.template").exists()
+    assert _git(repo, "status", "--porcelain").stdout == ""
+
+
+def test_success_message_reports_what_changed_not_what_was_restored(two_repos, capsys):
+    """Restoring every stack-owned path from MERGE_HEAD touches all of them
+    whether or not the upstream moved any, so the report has to ask git what
+    changed - otherwise a one-file update is announced as five."""
+    upstream, repo = two_repos
+    _write(upstream, "kb/CONTRACT.md", "stack kb contract v2\n")
+    _commit(upstream, "upstream changes exactly one stack path")
+
+    _merge()
+
+    out = capsys.readouterr().out
+    assert "Stack paths changed (1)" in out
+    assert "kb/CONTRACT.md" in out
+    assert "kb/CONVENTIONS.md.template" not in out
+
+
 def test_dirty_working_tree_is_refused_untouched(two_repos):
     upstream, repo = two_repos
     before = _git(repo, "rev-parse", "HEAD").stdout.strip()