torben
1b5ffea854
Files changed: - CHANGES.md - VERSION - docs/ownership-and-templates.md - instructions/gates.md - instructions/private-instance.md - tools/CONTRACT.md - tools/chemenu/commands/upstream_cmd.py - tools/chemenu/tests/test_upstream_cmd.py
388 lines
16 KiB
Python
388 lines
16 KiB
Python
"""`wikitool upstream` - take a stack update from a public upstream into a
|
|
private instance's `main` without letting the upstream's own content (a demo
|
|
corpus, a workshop run) ride along.
|
|
|
|
`git merge upstream/main` on its own treats a moved corpus dangerously
|
|
asymmetrically: a page the instance deleted and the upstream edited reports as
|
|
a conflict, a page the upstream *added* stages silently, and a page both sides
|
|
deleted is the only harmless case. `instructions/private-instance.md`'s prose
|
|
procedure closes that, by holding the merge open, forcing the content stages
|
|
(`ownership.CONTENT_STAGES`) back to the local side, and then restoring only
|
|
the paths `ownership.is_stack_owned` recognises as machinery. `upstream merge`
|
|
is that procedure in code, so the path set it acts on cannot drift from the
|
|
one `dist_cmd.py` ships - both read `chemenu.ownership` - and so a conflict in
|
|
the machinery layers, or a machinery file the upstream deleted, gets an
|
|
explained stop instead of a silently wrong commit.
|
|
|
|
`upstream verify` is the other half: given two revisions, did anything change
|
|
under a content stage except through a stack-owned path? It shares
|
|
`_content_leaks` with the postcheck `upstream merge` runs on itself, so a
|
|
hand-resolved merge or a future `dist upgrade` (Gitea #7) can be checked the
|
|
same way.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
import typer
|
|
|
|
from chemenu import config, ownership
|
|
from chemenu.commands import git_publish
|
|
from chemenu.commands._util import console, fail, success
|
|
|
|
app = typer.Typer(help="Take a stack update from a public upstream, machinery only.")
|
|
|
|
|
|
def _run(args: list[str]):
|
|
import subprocess
|
|
|
|
return subprocess.run(args, cwd=config.ROOT, capture_output=True, text=True)
|
|
|
|
|
|
def _rev_parse(rev: str) -> Optional[str]:
|
|
result = _run(["git", "rev-parse", "--verify", "-q", rev])
|
|
return result.stdout.strip() if result.returncode == 0 else None
|
|
|
|
|
|
def _git_dir() -> Optional[Path]:
|
|
result = _run(["git", "rev-parse", "--git-dir"])
|
|
if result.returncode != 0:
|
|
return None
|
|
path = Path(result.stdout.strip())
|
|
return path if path.is_absolute() else config.ROOT / path
|
|
|
|
|
|
def _working_tree_dirty() -> bool:
|
|
result = _run(["git", "status", "--porcelain"])
|
|
return bool(result.stdout.strip())
|
|
|
|
|
|
def _merge_in_progress() -> bool:
|
|
git_dir = _git_dir()
|
|
return git_dir is not None and (git_dir / "MERGE_HEAD").exists()
|
|
|
|
|
|
def _remote_resolves(remote: str) -> bool:
|
|
return _run(["git", "remote", "get-url", remote]).returncode == 0
|
|
|
|
|
|
def _is_ancestor(ancestor: str, of: str) -> bool:
|
|
return _run(["git", "merge-base", "--is-ancestor", ancestor, of]).returncode == 0
|
|
|
|
|
|
def _tree_has_path(rev: str, path: str) -> bool:
|
|
return _run(["git", "rev-parse", "--verify", "-q", f"{rev}:{path}"]).returncode == 0
|
|
|
|
|
|
def _tree_paths(rev: str) -> set[str]:
|
|
result = _run(["git", "ls-tree", "-r", "--name-only", "-z", rev])
|
|
if result.returncode != 0:
|
|
return set()
|
|
return {p for p in result.stdout.split("\0") if p}
|
|
|
|
|
|
def _content_leaks(since: str, until: str) -> list[str]:
|
|
"""Paths under a content stage that changed between `since` and `until`
|
|
through something other than a stack-owned path. Shared by `upstream
|
|
merge`'s own postcheck and `upstream verify`, so the two cannot disagree
|
|
about what a clean update looks like."""
|
|
result = _run(["git", "diff", "--name-only", "-z", since, until, "--", *ownership.CONTENT_STAGES])
|
|
if result.returncode != 0:
|
|
fail(
|
|
f"`git diff {since} {until}` failed - is {since} a revision in this repository?\n"
|
|
f"{result.stderr}"
|
|
)
|
|
return []
|
|
changed = [p for p in result.stdout.split("\0") if p]
|
|
return sorted(p for p in changed if not ownership.is_stack_owned(p))
|
|
|
|
|
|
def _stack_paths_changed(since: str, until: str) -> list[str]:
|
|
"""The subset of the same diff that *is* a stack-owned path - the paths
|
|
that legitimately moved, for the success message."""
|
|
result = _run(["git", "diff", "--name-only", "-z", since, until, "--", *ownership.CONTENT_STAGES])
|
|
changed = [p for p in result.stdout.split("\0") if p]
|
|
return sorted(p for p in changed if ownership.is_stack_owned(p))
|
|
|
|
|
|
# --- upstream merge ---------------------------------------------------------
|
|
|
|
|
|
def _prune_empty_dirs(stage: str) -> None:
|
|
"""Remove directories left empty under `stage` after tracked files were
|
|
deleted. git tracks no directories, so an emptied one is invisible to
|
|
`git status` and would otherwise linger in the working tree as litter -
|
|
an empty `kb/<area>/` that only ever existed in the upstream's corpus.
|
|
Never touches a directory that still holds anything, ignored files
|
|
included."""
|
|
stage_dir = config.ROOT / stage
|
|
if not stage_dir.is_dir():
|
|
return
|
|
for path in sorted(stage_dir.rglob("*"), key=lambda p: len(p.parts), reverse=True):
|
|
if path.is_dir() and not any(path.iterdir()):
|
|
path.rmdir()
|
|
|
|
|
|
def _restore_stage_to_local(stage: str, tracked_paths: set[str]) -> None:
|
|
"""Force one content stage back to the local (HEAD) side, whatever the
|
|
merge did to it.
|
|
|
|
Deletes **only what git tracks on either side** - never the stage
|
|
directory wholesale. That distinction is the whole point of this function:
|
|
`reports/` is gitignored except its contract (see .gitignore), so a
|
|
content stage's working tree legitimately holds local data that is not in
|
|
any tree and not recomputable - the telemetry traces `eval score` reads,
|
|
saved eval reports, past lint reports. A blanket `rm -rf` of the stage
|
|
takes all of it out as collateral for a merge that was never about it.
|
|
|
|
Handles a stage that exists only in MERGE_HEAD too (the upstream
|
|
introduced it): what the merge wrote is removed, and there is simply
|
|
nothing to check out from HEAD afterwards.
|
|
"""
|
|
prefix = f"{stage}/"
|
|
stage_paths = [p for p in tracked_paths if p.startswith(prefix)]
|
|
if not stage_paths:
|
|
return
|
|
|
|
_run(["git", "rm", "-rq", "--cached", "--ignore-unmatch", stage])
|
|
for relative in stage_paths:
|
|
target = config.ROOT / relative
|
|
if target.is_file() or target.is_symlink():
|
|
target.unlink()
|
|
_prune_empty_dirs(stage)
|
|
if _tree_has_path("HEAD", stage):
|
|
_run(["git", "checkout", "HEAD", "--", stage])
|
|
|
|
|
|
def _remote_gate_warning() -> None:
|
|
if git_publish.read_allowed_push_urls() is not None:
|
|
return
|
|
console.print(
|
|
"[bold yellow]WARN[/bold yellow] No .wikitool-remotes.json in this checkout - the "
|
|
"Publish-Remote Gate is unarmed, so a future `publish` to the wrong remote would not "
|
|
"be caught. `upstream merge` never pushes and proceeds regardless, but a checkout that "
|
|
"takes stack updates from a public upstream should arm the gate before its next publish "
|
|
"- see instructions/private-instance.md step 4."
|
|
)
|
|
|
|
|
|
def _precondition_failure(remote: str) -> Optional[str]:
|
|
if _working_tree_dirty():
|
|
return (
|
|
"Working tree is not clean (`git status --porcelain` printed something). "
|
|
"`upstream merge` refuses to start on a dirty tree so a refusal never has to "
|
|
"guess which changes were already there. Commit or stash first."
|
|
)
|
|
if _merge_in_progress():
|
|
return (
|
|
"A merge is already in progress (.git/MERGE_HEAD exists). Resolve or abort it "
|
|
"(`git merge --abort`) before running `upstream merge`."
|
|
)
|
|
if not _remote_resolves(remote):
|
|
return f"Remote '{remote}' does not resolve (`git remote get-url {remote}` failed)."
|
|
return None
|
|
|
|
|
|
def _unresolved_conflict_message(unresolved: list[str], remote: str, branch: str) -> str:
|
|
listed = "\n".join(f" - {p}" for p in unresolved)
|
|
return (
|
|
f"A real conflict remains in the machinery layers after restoring the content stages "
|
|
f"and the stack-owned paths from {remote}/{branch}:\n{listed}\n\n"
|
|
"The merge is left open, uncommitted - nothing was written to the branch. Per "
|
|
"instructions/private-instance.md's decision points: this means the checkout changed "
|
|
"the stack locally, which private instances do not do. Take the upstream side for "
|
|
"these paths (`git checkout --theirs -- <path>` then `git add`) and re-file the local "
|
|
"change as an issue against the public repo, or resolve deliberately and "
|
|
"`git commit --no-edit` yourself. `git merge --abort` gives up the merge entirely."
|
|
)
|
|
|
|
|
|
def _postcheck_failure_message(leaks: list[str], before: str) -> str:
|
|
listed = "\n".join(f" - {p}" for p in leaks)
|
|
return (
|
|
f"The merge commit exists (content stages are not what they were before this ran), "
|
|
f"but it changed content outside of a stack-owned path:\n{listed}\n\n"
|
|
f"This was NOT rolled back - the state belongs in front of you, not behind an automatic "
|
|
f"repair the command applies to itself. Compare against the pre-merge commit ({before}) "
|
|
"and decide by hand whether to revert the merge commit, cherry-pick around it, or fix "
|
|
"forward. This is a bug in `upstream merge` or in `ownership.is_stack_owned` if it "
|
|
"reproduces - please report it rather than working around it silently."
|
|
)
|
|
|
|
|
|
def _merge_success_message(
|
|
changed: list[str], deleted: list[str], remote: str, branch: str
|
|
) -> str:
|
|
"""What the merge actually did, measured against the pre-merge commit
|
|
rather than against what was restored.
|
|
|
|
`changed` is the real diff - restoring every stack-owned path from
|
|
MERGE_HEAD touches each of them whether or not the upstream moved any, so
|
|
reporting the restore list would claim seven updates for a merge that
|
|
changed one file, and a reader who checks would find the report wrong.
|
|
"""
|
|
deleted_set = set(deleted)
|
|
lines = [
|
|
f"Merged {remote}/{branch}. Content stages "
|
|
f"({', '.join(ownership.CONTENT_STAGES)}) are unchanged."
|
|
]
|
|
if changed:
|
|
lines.append(f"Stack paths changed ({len(changed)}):")
|
|
lines += [
|
|
f" - {p}" + (" (deleted, following the upstream)" if p in deleted_set else "")
|
|
for p in changed
|
|
]
|
|
else:
|
|
lines.append("No stack-owned path changed.")
|
|
return "\n".join(lines)
|
|
|
|
|
|
@app.command("merge")
|
|
def merge_command(
|
|
remote: str = typer.Option("upstream", "--remote", help="Remote to merge from"),
|
|
branch: str = typer.Option("main", "--branch", help="Branch to merge"),
|
|
no_fetch: bool = typer.Option(
|
|
False, "--no-fetch", help="Skip `git fetch <remote>` - use whatever is already fetched"
|
|
),
|
|
):
|
|
"""Merge `<remote>/<branch>` into the current branch, machinery only:
|
|
every path under a content stage (kb/, raw/, work/, reports/) is forced
|
|
back to the local side except a stack-owned path (`<stage>/CONTRACT.md`,
|
|
or anything ending `.template` under a content stage), which is taken
|
|
from the upstream - including a deletion, if the upstream removed one. A
|
|
real conflict elsewhere (tools/, types/, instructions/) leaves the merge
|
|
open and unresolved rather than guessing. Not idempotent: it can leave an
|
|
open merge behind on refusal. See instructions/private-instance.md."""
|
|
problem = _precondition_failure(remote)
|
|
if problem:
|
|
fail(problem)
|
|
return
|
|
|
|
_remote_gate_warning()
|
|
|
|
before = _rev_parse("HEAD")
|
|
if before is None:
|
|
fail("HEAD does not resolve - is this a git repository with at least one commit?")
|
|
return
|
|
|
|
if not no_fetch:
|
|
fetch_result = _run(["git", "fetch", remote, branch])
|
|
if fetch_result.returncode != 0:
|
|
fail(f"`git fetch {remote} {branch}` failed:\n{fetch_result.stderr}")
|
|
return
|
|
|
|
remote_ref = f"{remote}/{branch}"
|
|
if _rev_parse(remote_ref) is None:
|
|
fail(f"'{remote_ref}' does not resolve - fetch it first, or check --remote/--branch.")
|
|
return
|
|
|
|
if _is_ancestor(remote_ref, "HEAD"):
|
|
success(f"Already up to date with {remote_ref}.")
|
|
return
|
|
|
|
# The exit code is deliberately not the test - conflicts under the content
|
|
# stages are expected here and are exactly what the next steps undo. What
|
|
# *is* load-bearing is that a merge actually opened: without MERGE_HEAD,
|
|
# `_tree_paths("MERGE_HEAD")` is empty, and every stack-owned path in HEAD
|
|
# would then read as "the upstream deleted it" and be removed. A merge git
|
|
# refused to start (unrelated histories, an ignored file in the way) must
|
|
# therefore stop here, with the tree untouched.
|
|
merge_result = _run(["git", "merge", "--no-commit", "--no-ff", remote_ref])
|
|
if not _merge_in_progress():
|
|
fail(
|
|
f"`git merge --no-commit --no-ff {remote_ref}` did not open a merge, so there is "
|
|
f"nothing to scope - the working tree is unchanged:\n"
|
|
f"{merge_result.stdout}{merge_result.stderr}"
|
|
)
|
|
return
|
|
|
|
merge_head_paths = _tree_paths("MERGE_HEAD")
|
|
head_paths = _tree_paths("HEAD")
|
|
tracked_paths = merge_head_paths | head_paths
|
|
|
|
for stage in ownership.CONTENT_STAGES:
|
|
_restore_stage_to_local(stage, tracked_paths)
|
|
|
|
stack_paths = sorted(
|
|
p for p in (merge_head_paths | head_paths) if ownership.is_stack_owned(p)
|
|
)
|
|
|
|
# Only the deletions are recorded: what was *restored* is every stack-owned
|
|
# path in MERGE_HEAD, which is not the same question as what changed - the
|
|
# success message asks git for that instead.
|
|
deleted: list[str] = []
|
|
for relative in stack_paths:
|
|
if relative in merge_head_paths:
|
|
checkout = _run(["git", "checkout", "MERGE_HEAD", "--", relative])
|
|
if checkout.returncode != 0:
|
|
fail(
|
|
f"`git checkout MERGE_HEAD -- {relative}` failed even though it is listed "
|
|
f"in MERGE_HEAD's own tree:\n{checkout.stderr}\nThe merge is left open."
|
|
)
|
|
return
|
|
else:
|
|
_run(["git", "rm", "-q", "--cached", "--ignore-unmatch", relative])
|
|
target = config.ROOT / relative
|
|
if target.exists():
|
|
target.unlink()
|
|
deleted.append(relative)
|
|
|
|
unresolved = [p for p in _run(["git", "diff", "--name-only", "--diff-filter=U"]).stdout.splitlines() if p]
|
|
if unresolved:
|
|
fail(_unresolved_conflict_message(unresolved, remote, branch))
|
|
return
|
|
|
|
commit_result = _run(["git", "commit", "--no-edit"])
|
|
if commit_result.returncode != 0:
|
|
fail(f"`git commit --no-edit` failed:\n{commit_result.stderr}")
|
|
return
|
|
|
|
leaks = _content_leaks(before, "HEAD")
|
|
if leaks:
|
|
fail(_postcheck_failure_message(leaks, before))
|
|
return
|
|
|
|
success(
|
|
_merge_success_message(_stack_paths_changed(before, "HEAD"), deleted, remote, branch)
|
|
)
|
|
|
|
|
|
# --- upstream verify ---------------------------------------------------------
|
|
|
|
|
|
def _verify_failure_message(leaks: list[str], since: str, until: str) -> str:
|
|
listed = "\n".join(f" - {p}" for p in leaks)
|
|
return (
|
|
f"Content under a content stage (kb/, raw/, work/, reports/) changed between {since} "
|
|
f"and {until} through a path that is not stack-owned:\n{listed}\n\n"
|
|
"That is upstream content (or an equivalent local change) that reached this range "
|
|
"outside of a stack-owned path - inspect it before trusting this range as machinery-only."
|
|
)
|
|
|
|
|
|
def _verify_success_message(stack_moved: list[str], since: str, until: str) -> str:
|
|
if not stack_moved:
|
|
return f"No content changed between {since} and {until} under kb/, raw/, work/, reports/."
|
|
listed = "\n".join(f" - {p}" for p in stack_moved)
|
|
return (
|
|
f"Clean: only stack-owned paths changed under kb/, raw/, work/, reports/ between "
|
|
f"{since} and {until}:\n{listed}"
|
|
)
|
|
|
|
|
|
@app.command("verify")
|
|
def verify_command(
|
|
since: str = typer.Option(..., "--since", help="Git revision to compare from"),
|
|
until: str = typer.Option("HEAD", "--until", help="Git revision to compare to"),
|
|
):
|
|
"""Check that nothing under a content stage changed between --since and
|
|
--until except through a stack-owned path. Read-only, and exempt from the
|
|
Iteration Budget Gate - the same treatment `migrate verify` gets, for the
|
|
same reason: a check an agent has to ration is a check that gets skipped."""
|
|
leaks = _content_leaks(since, until)
|
|
if leaks:
|
|
fail(_verify_failure_message(leaks, since, until))
|
|
return
|
|
success(_verify_success_message(_stack_paths_changed(since, until), since, until))
|