fix: publish/sync merge generated files mechanically and carry non-overlapping uncommitted work through a rebase (#180)

Overlap only in kb/index.md, kb/log.md, kb/provenance.md and kb/**/INDEX.md no longer
fails a reconcile or reaches the rebase-review gate: the log keeps both sides' entries,
the catalog and provenance are regenerated. Uncommitted work no incoming commit touches
rides through the rebase via --autostash; the working tree is backed up under
refs/wikitool/reconcile-backup first. is_generated is narrowed to kb/.

Files changed:
- CHANGES.md
- VERSION
- instructions/gates.md
- instructions/session-setup.md
- tools/CONTRACT.md
- tools/chemenu/commands/git_publish.py
- tools/chemenu/commands/index_build.py
- tools/chemenu/commands/provenance_cmd.py
- tools/chemenu/tests/test_git_publish.py

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2

tools/chemenu/tests/test_git_publish.py

@@ -1,4 +1,5 @@
 import json
+import shutil
 import subprocess
 
 import pytest
@@ -367,8 +368,8 @@ def _publish(**overrides):
     """Call `publish_command` directly. Every parameter must be given a real
     value: bypassing Typer's CLI parsing means an omitted argument keeps its
     `typer.Option(...)` sentinel instead of the value it wraps."""
-    kwargs = dict(message="change", push=True, confirm=None, yes=False, threshold=10,
-                  remote="origin", branch="main", path=None)
+    kwargs = dict(message="change", push=True, confirm=None, confirm_rebase=None, yes=False,
+                  threshold=10, remote="origin", branch="main", path=None)
     kwargs.update(overrides)
     publish_command(**kwargs)
 
@@ -515,6 +516,14 @@ def test_generated_files_are_recognised_wherever_they_sit():
     assert not is_generated("kb/concepts/protocols/Modbus.md")
 
 
+def test_an_index_md_outside_kb_is_not_generated():
+    """A captured repository under raw/ may carry its own INDEX.md. It is source material, and
+    `reconcile` discards and regenerates whatever this predicate calls generated."""
+    assert not is_generated("raw/repos/project/INDEX.md")
+    assert not is_generated("docs/INDEX.md")
+    assert counted_files(["raw/repos/project/INDEX.md"]) == ["raw/repos/project/INDEX.md"]
+
+
 def test_paths_land_in_the_group_a_reviewer_expects():
     assert group_of("kb/concepts/X.md")[0] == "Published knowledge"
     assert group_of("AGENTS.md")[0] == "Agent control plane"
@@ -1497,3 +1506,395 @@ def test_a_mass_update_token_does_not_depend_on_the_message(repo):
     assert _git(
         repo, "log", "-1", "--format=%(trailers:key=Co-Authored-By,valueonly)"
     ).stdout.strip() == "A <a@x>"
+
+
+# --- Reconciling around generated files: two ingests on two checkouts -----------------------
+#
+# Every ingest touches kb/index.md and kb/log.md. These tests build a real catalog with the
+# shipped rebuild functions, so "the catalog is current" is checked against what `index
+# rebuild`/`sources rebuild-index` actually write, not against a hand-made stand-in.
+
+from contextlib import contextmanager
+
+from chemenu.commands.log_append import format_log_entry, parse_log_entries
+from chemenu.frontmatter_io import write_page
+from chemenu.tests.conftest import use_shipped_type_specs
+
+GENERATED_FILES = ("kb/index.md", "kb/log.md", "kb/provenance.md", "kb/concepts/INDEX.md")
+
+
+@contextmanager
+def _at(root):
+    """Point `config.ROOT` at another checkout for the duration - the writer clone builds its
+    catalog with the same functions, against its own tree."""
+    previous = config.ROOT
+    config.ROOT = root
+    try:
+        yield
+    finally:
+        config.ROOT = previous
+
+
+def _write_concept(root, title, text="A concept.\n"):
+    write_page(
+        root / f"kb/concepts/protocols/{title}.md",
+        {"type": "types/concept.md", "concept_type": "protocol", "tags": [],
+         "created": "2026-10-06", "modified": "2026-10-06", "related": [], "sources": []},
+        f"\n# {title}\n\n## Definition\n\n{text}",
+    )
+
+
+def _ingest(root, title, text="A concept.\n"):
+    """What an ingest leaves in the working tree: a page, the regenerated catalog and
+    provenance, and one appended log entry."""
+    _write_concept(root, title, text)
+    with _at(root):
+        git_publish.regenerate_generated()
+    with (root / "kb/log.md").open("a", encoding="utf-8", newline="\n") as log:
+        log.write("\n" + format_log_entry("ingest", title, today="2026-10-06"))
+
+
+def _commit_all(root, message):
+    _git(root, "add", "-A")
+    _git(root, "commit", "-m", message)
+
+
+@pytest.fixture
+def wiki_repo(repo, monkeypatch):
+    """`repo` with a real, pushed kb/: one collection, one page, its catalog and provenance
+    built by the shipped functions, and a log."""
+    use_shipped_type_specs(monkeypatch)
+    (repo / "kb/concepts").mkdir(parents=True)
+    (repo / "kb/concepts/COLLECTION.md").write_text(
+        "---\nprofile: concepts\nrequired_by_stack: false\noutbound:\n  any: [see-also]\n---\n\n"
+        "# kb/concepts/ - Collection Contract\n",
+        encoding="utf-8",
+    )
+    (repo / "raw/notes").mkdir(parents=True)
+    (repo / "raw/notes/a.md").write_text("raw\n", encoding="utf-8")
+    (repo / "kb/log.md").write_text("# Wiki Log\n", encoding="utf-8")
+    _ingest(repo, "Base")
+    _commit_all(repo, "wiki")
+    _git(repo, "push", "origin", "main")
+    return repo
+
+
+def _writer_ingest(writer, title):
+    _ingest(writer, title)
+    _commit_all(writer, f"writer ingest: {title}")
+    _git(writer, "push", "origin", "main")
+
+
+def _tree(root):
+    """Every file of the working tree outside .git, as bytes."""
+    return {
+        path.relative_to(root).as_posix(): path.read_bytes()
+        for path in root.rglob("*")
+        if path.is_file() and ".git" not in path.relative_to(root).parts
+    }
+
+
+def _non_generated(tree):
+    return {path: data for path, data in tree.items() if not is_generated(path)}
+
+
+def _assert_kept(before, root):
+    """Every non-generated file `before` holds is still there, byte for byte. Files the
+    incoming commits add are new, not changed."""
+    after = _tree(root)
+    assert {path: after.get(path) for path in _non_generated(before)} == _non_generated(before)
+
+
+def _check_tree(root):
+    """A fresh clone of the remote tip, as a place to compare its generated files against a
+    rebuild of that same tip."""
+    check = root.parent / "check"
+    shutil.rmtree(check, ignore_errors=True)
+    subprocess.run(["git", "clone", str(_remote_for(root)), str(check)], check=True,
+                   capture_output=True)
+    return check
+
+
+def _assert_generated_current(root):
+    """The generated files at `root` are byte for byte what a rebuild of `root` writes today."""
+    before = {path: (root / path).read_bytes() for path in GENERATED_FILES if path != "kb/log.md"}
+    with _at(root):
+        git_publish.regenerate_generated()
+    after = {path: (root / path).read_bytes() for path in before}
+    assert before == after
+
+
+def _logged_titles(root):
+    return [title for _date, _op, title in parse_log_entries(
+        (root / "kb/log.md").read_text(encoding="utf-8"))]
+
+
+def test_case_1_an_uncommitted_ingest_publishes_over_an_ingest_that_landed_meanwhile(wiki_repo):
+    writer = _clone_writer(wiki_repo)
+    _writer_ingest(writer, "Writer")
+    _ingest(wiki_repo, "Local")
+
+    _publish(message="ingest: Local")  # must not raise - no gate, no failure
+
+    check = _check_tree(wiki_repo)
+    assert (check / "kb/concepts/protocols/Writer.md").is_file()
+    assert (check / "kb/concepts/protocols/Local.md").is_file()
+    assert _logged_titles(check) == ["Base", "Writer", "Local"]
+    _assert_generated_current(check)
+    assert _status(wiki_repo) == ""
+
+
+def test_case_1_on_sync_leaves_the_regenerated_catalog_uncommitted(wiki_repo):
+    writer = _clone_writer(wiki_repo)
+    _writer_ingest(writer, "Writer")
+    _ingest(wiki_repo, "Local")
+    page = (wiki_repo / "kb/concepts/protocols/Local.md").read_bytes()
+
+    outcome = reconcile("origin", "main", None)
+
+    assert (outcome.status, outcome.regenerated) == ("fast-forwarded", True)
+    assert (wiki_repo / "kb/concepts/protocols/Local.md").read_bytes() == page
+    assert _logged_titles(wiki_repo) == ["Base", "Writer", "Local"]
+    _assert_generated_current(wiki_repo)
+    assert "kb/index.md" in _status(wiki_repo)
+    assert subprocess.run(["git", "show-ref", "--verify", "-q", git_publish.BACKUP_REF],
+                          cwd=wiki_repo).returncode != 0  # dropped on success
+
+
+def test_case_2_two_committed_ingests_rebase_without_the_review_gate(wiki_repo):
+    writer = _clone_writer(wiki_repo)
+    _writer_ingest(writer, "Writer")
+    _ingest(wiki_repo, "Local")
+    _commit_all(wiki_repo, "local ingest, push failed")  # the stranded commit
+
+    _publish(message="retry")  # must not raise
+
+    check = _check_tree(wiki_repo)
+    assert _logged_titles(check) == ["Base", "Writer", "Local"]
+    _assert_generated_current(check)
+    assert _status(wiki_repo) == ""
+
+
+def test_case_2_on_sync_ends_with_exit_0_and_both_sides_in_the_log(wiki_repo):
+    writer = _clone_writer(wiki_repo)
+    _writer_ingest(writer, "Writer")
+    _ingest(wiki_repo, "Local")
+    _commit_all(wiki_repo, "local ingest")
+
+    _sync()  # must not raise
+
+    assert _logged_titles(wiki_repo) == ["Base", "Writer", "Local"]
+    _assert_generated_current(wiki_repo)
+    assert "writer ingest: Writer" in _git(wiki_repo, "log", "--oneline").stdout
+
+
+def test_case_2_still_gates_on_an_overlapping_page_and_names_no_generated_file(wiki_repo):
+    writer = _clone_writer(wiki_repo)
+    _ingest(writer, "Base", text="Writer's definition.\n")
+    _commit_all(writer, "writer edits Base")
+    _git(writer, "push", "origin", "main")
+    _ingest(wiki_repo, "Base", text="Local definition.\n")
+    _commit_all(wiki_repo, "local edits Base")
+
+    outcome = reconcile("origin", "main", None)
+
+    assert outcome.status == "needs-review"
+    assert outcome.overlap_files == ["kb/concepts/protocols/Base.md"]
+    assert "kb/log.md" not in outcome.overlap_diff and "kb/index.md" not in outcome.overlap_diff
+
+
+def test_case_2_in_the_retry_path_commits_the_regenerated_catalog_before_the_push(
+    wiki_repo, monkeypatch
+):
+    writer = _clone_writer(wiki_repo)
+    raced = {"done": False}
+    real_run = git_publish._run
+
+    def racy_run(args, **kwargs):
+        if args[:2] == ["git", "push"] and not raced["done"]:
+            raced["done"] = True
+            _writer_ingest(writer, "Writer")
+        return real_run(args, **kwargs)
+
+    monkeypatch.setattr(git_publish, "_run", racy_run)
+    _ingest(wiki_repo, "Local")
+
+    _publish(message="ingest: Local")  # must not raise
+
+    assert raced["done"]
+    check = _check_tree(wiki_repo)
+    assert _logged_titles(check) == ["Base", "Writer", "Local"]
+    _assert_generated_current(check)
+    assert "chore: regenerate the catalog" in _git(check, "log", "-1", "--format=%s").stdout
+    assert _status(wiki_repo) == ""
+
+
+def _stranded_ingest_plus_uncommitted_ingest(wiki_repo):
+    writer = _clone_writer(wiki_repo)
+    _writer_ingest(writer, "Writer")
+    _ingest(wiki_repo, "Stranded")
+    _commit_all(wiki_repo, "stranded ingest")
+    _ingest(wiki_repo, "Uncommitted")
+    (wiki_repo / "README.md").write_text("init\nedited, not committed\n", encoding="utf-8")
+
+
+def test_case_3_publish_carries_uncommitted_work_over_a_stranded_commit(wiki_repo):
+    _stranded_ingest_plus_uncommitted_ingest(wiki_repo)
+
+    _publish(message="ingest: Uncommitted")  # must not raise
+
+    check = _check_tree(wiki_repo)
+    assert (check / "kb/concepts/protocols/Uncommitted.md").is_file()
+    assert (check / "README.md").read_text(encoding="utf-8") == "init\nedited, not committed\n"
+    assert _logged_titles(check) == ["Base", "Writer", "Stranded", "Uncommitted"]
+    _assert_generated_current(check)
+
+
+def test_case_3_sync_leaves_the_uncommitted_pages_exactly_as_they_were(wiki_repo):
+    _stranded_ingest_plus_uncommitted_ingest(wiki_repo)
+    before = _tree(wiki_repo)
+
+    _sync()  # must not raise
+
+    _assert_kept(before, wiki_repo)
+    status = _status(wiki_repo)
+    assert "README.md" in status and "Uncommitted.md" in status  # still uncommitted
+    assert _logged_titles(wiki_repo) == ["Base", "Writer", "Stranded", "Uncommitted"]
+    _assert_generated_current(wiki_repo)
+
+
+def test_case_3_without_generated_files_rebases_and_rebuilds_nothing(repo, monkeypatch):
+    """A stranded commit and an unrelated uncommitted edit: `git rebase` used to refuse over
+    the edit. Nothing generated is involved, so nothing is regenerated."""
+    monkeypatch.setattr(git_publish, "regenerate_generated",
+                        lambda *a, **k: pytest.fail("nothing generated was involved"))
+    writer = _clone_writer(repo)
+    _push_from_writer(writer, "from-writer.md", "writer\n")
+    (repo / "local.md").write_text("local\n", encoding="utf-8")
+    _commit_all(repo, "stranded")
+    (repo / "README.md").write_text("uncommitted\n", encoding="utf-8")
+
+    outcome = reconcile("origin", "main", None)
+
+    assert (outcome.status, outcome.regenerated) == ("rebased", False)
+    assert (repo / "README.md").read_text(encoding="utf-8") == "uncommitted\n"
+    assert (repo / "from-writer.md").is_file()
+
+
+@pytest.mark.parametrize("how", ["raises", "fails"])
+def test_uncommitted_work_survives_an_abort_in_the_middle_of_the_rebase(wiki_repo, monkeypatch, how):
+    """The invariant: whatever happens mid-rebase, every non-generated file in the working tree
+    is byte-identical afterwards, and the branch has not moved. A failure also restores the
+    generated files and names the backup ref."""
+    _stranded_ingest_plus_uncommitted_ingest(wiki_repo)
+    before_tree = _tree(wiki_repo)
+    before_head = _git(wiki_repo, "rev-parse", "HEAD").stdout
+
+    def forced(*_args):
+        if how == "raises":
+            raise RuntimeError("forced abort mid-rebase")
+        return "forced failure"
+
+    monkeypatch.setattr(git_publish, "_resolve_generated_conflicts", forced)
+
+    if how == "raises":
+        with pytest.raises(RuntimeError):
+            reconcile("origin", "main", None)
+    else:
+        outcome = reconcile("origin", "main", None)
+        assert outcome.status == "conflict"
+        assert git_publish.BACKUP_REF in outcome.detail
+        assert _tree(wiki_repo) == before_tree
+
+    assert _non_generated(_tree(wiki_repo)) == _non_generated(before_tree)
+    assert _git(wiki_repo, "rev-parse", "HEAD").stdout == before_head
+    assert not git_publish._rebase_in_progress()
+    assert _git(wiki_repo, "stash", "list").stdout == ""
+    # The backup holds the working tree as it stood, uncommitted page included.
+    assert "Uncommitted" in _git(
+        wiki_repo, "show", f"{git_publish.BACKUP_REF}:kb/concepts/protocols/Uncommitted.md"
+    ).stdout
+
+
+def _assert_untouched(repo, run):
+    tree, head = _tree(repo), _git(repo, "rev-parse", "HEAD").stdout
+    with pytest.raises(typer.Exit) as excinfo:
+        run()
+    assert excinfo.value.exit_code == 1
+    assert _tree(repo) == tree
+    assert _git(repo, "rev-parse", "HEAD").stdout == head
+
+
+def test_a_hand_edited_log_is_left_to_git_as_before(wiki_repo):
+    writer = _clone_writer(wiki_repo)
+    _writer_ingest(writer, "Writer")
+    _ingest(wiki_repo, "Local")
+    log = wiki_repo / "kb/log.md"
+    log.write_text(log.read_text(encoding="utf-8").replace("# Wiki Log", "# Edited Log"),
+                   encoding="utf-8")
+
+    _assert_untouched(wiki_repo, _sync)
+
+
+def test_a_log_rewritten_in_a_local_commit_aborts_the_rebase_as_before(wiki_repo):
+    writer = _clone_writer(wiki_repo)
+    _writer_ingest(writer, "Writer")
+    _ingest(wiki_repo, "Local")
+    log = wiki_repo / "kb/log.md"
+    log.write_text(log.read_text(encoding="utf-8").replace("# Wiki Log", "# Edited Log"),
+                   encoding="utf-8")
+    _commit_all(wiki_repo, "local ingest with a rewritten log")
+
+    _assert_untouched(wiki_repo, _sync)
+
+
+def test_an_uncommitted_change_to_a_file_the_incoming_commits_change_stops_as_before(wiki_repo):
+    writer = _clone_writer(wiki_repo)
+    _ingest(writer, "Base", text="Writer's definition.\n")
+    _commit_all(writer, "writer edits Base")
+    _git(writer, "push", "origin", "main")
+    _ingest(wiki_repo, "Stranded")
+    _commit_all(wiki_repo, "stranded")
+    _write_concept(wiki_repo, "Base", "Uncommitted local definition.\n")
+
+    _assert_untouched(wiki_repo, _sync)
+
+
+def test_an_untracked_file_on_an_incoming_path_stops_as_before(wiki_repo):
+    writer = _clone_writer(wiki_repo)
+    _writer_ingest(writer, "Writer")
+    _ingest(wiki_repo, "Stranded")
+    _commit_all(wiki_repo, "stranded")
+    _write_concept(wiki_repo, "Writer", "Same title, written here and never committed.\n")
+
+    _assert_untouched(wiki_repo, _sync)
+
+
+def test_a_plain_fast_forward_regenerates_nothing(wiki_repo, monkeypatch):
+    """None of the three cases: the result is what it always was, and in particular no rebuild
+    runs - it would stamp today's date into kb/index.md and leave every sync dirty."""
+    writer = _clone_writer(wiki_repo)
+    _writer_ingest(writer, "Writer")
+    monkeypatch.setattr(git_publish, "regenerate_generated",
+                        lambda *a, **k: pytest.fail("no rebuild without a collision"))
+
+    outcome = reconcile("origin", "main", None)
+
+    assert (outcome.status, outcome.regenerated) == ("fast-forwarded", False)
+    assert _status(wiki_repo) == ""
+
+
+def test_an_index_md_under_raw_still_goes_to_review_when_both_sides_change_it(repo):
+    (repo / "raw/repos/project").mkdir(parents=True)
+    (repo / "raw/repos/project/INDEX.md").write_text("original\n", encoding="utf-8")
+    _commit_all(repo, "capture")
+    _git(repo, "push", "origin", "main")
+    writer = _clone_writer(repo)
+    _push_from_writer(writer, "raw/repos/project/INDEX.md", "writer\n")
+    (repo / "raw/repos/project/INDEX.md").write_text("local\n", encoding="utf-8")
+    _commit_all(repo, "local")
+
+    outcome = reconcile("origin", "main", None)
+
+    assert outcome.status == "needs-review"
+    assert outcome.overlap_files == ["raw/repos/project/INDEX.md"]