torben/chemenu
1
1
Fork 0
Code Issues 22 Pull Requests Actions Packages Releases 35 Activity

feat: dist upgrade - apply a stack update, not just detect one (#7)
CI / verify (push) Successful in 54s
Details
Release / release (push) Successful in 34s
Details

Browse Source 
Files changed:
- CHANGES.md
- INSTALL.md
- VERSION
- tools/CONTRACT.md
- tools/chemenu/commands/dist_cmd.py
- tools/chemenu/kb_state.py
- tools/chemenu/ownership.py
- tools/chemenu/tests/test_dist_upgrade.py
This commit is contained in:
Torben Nehmer
2026-09-04 11:36:21 +02:00
parent 1b5ffea854
commit cd81ba3d4f
8 changed files with 988 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/chemenu/commands/dist_cmd.py
+409 -1
View File
@@ -33,14 +33,20 @@ import hashlib
import json
import os
import re
import shutil
import stat
import subprocess
import tarfile
import tempfile
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, NamedTuple, Optional, Union
import typer
from chemenu import config, conventions, kb_collections, kb_state, ownership, version as version_mod
from chemenu.commands._util import fail, rel_path, success, today_iso
from chemenu.commands._util import console, fail, rel_path, success, today_iso
app = typer.Typer(help="Build a distributable copy of the wiki machinery.")
@@ -581,3 +587,405 @@ def run_export(target: Path, dry_run: bool = False, origin: Optional[Origin] = N
_write_plan(target, plan)
success(f"Exported {len(plan)} file(s) to {rel_path(target)}.")
# --- dist upgrade ------------------------------------------------------------
#
# Apply a release `dist export` produced, rather than merely detecting one
# (`version check`). Gitea #7 has the full design; the short version: the
# write set is exactly the *new* stamp's `files` block, minus the paths an
# export seeds once and the instance owns from then on
# (`ownership.is_export_stub`, `ownership.is_upgrade_preserved`), plus the
# stamp itself. Every candidate path is classified against the *old* stamp's
# recorded digest - unchanged, locally modified, or locally deleted - and a
# modified/deleted file is never silently overwritten. This never calls a
# release feed; the caller supplies an already-downloaded tree or archive.
@dataclass(frozen=True)
class FileClassification:
"""The four-way split of every path `dist upgrade` would touch, plus the
fifth direction (`removed`) that has no write set of its own."""
unchanged: list[str]
modified: list[str]
deleted: list[str]
new: list[str]
removed: list[str]
@property
def blocked(self) -> list[str]:
"""Locally changed paths - modified or deleted - which are never
silently overwritten."""
return sorted(self.modified + self.deleted)
def _write_candidates(new_files: dict) -> set[str]:
"""Every path `dist upgrade` may write, from the new stamp's `files`
block: everything except the paths an export re-seeds from a blank
template every time (`ownership.is_export_stub`) and the paths an export
seeds once and the instance owns afterward (`ownership.is_upgrade_preserved`).
The release stamp itself is added separately - it is never a member of its
own `files` block, see `build_stamp`."""
return {
relative
for relative in new_files
if not ownership.is_export_stub(Path(relative).name)
and not ownership.is_upgrade_preserved(relative)
}
def _classify_files(old_files: dict, new_files: dict) -> FileClassification:
candidates = _write_candidates(new_files)
recorded_for_candidates = {r: d for r, d in old_files.items() if r in candidates}
statuses = kb_state.compare_against_stamp(recorded_for_candidates)
unchanged: list[str] = []
modified: list[str] = []
deleted: list[str] = []
new: list[str] = []
for relative in sorted(candidates):
if relative not in old_files:
new.append(relative)
continue
status = statuses[relative]
if status == kb_state.UNCHANGED:
unchanged.append(relative)
elif status == kb_state.MODIFIED:
modified.append(relative)
else:
deleted.append(relative)
removed = sorted(set(old_files) - set(new_files))
return FileClassification(unchanged, modified, deleted, new, removed)
def _verify_sha256_sidecar(archive: Path) -> None:
"""WARN, never fail, on a missing sidecar - only a corrupted one that
*is* present is a reason to stop, per Gitea #7's design table."""
sidecar = archive.with_name(archive.name + ".sha256")
if not sidecar.is_file():
console.print(
f"[yellow]WARN[/yellow] No {sidecar.name} beside {archive.name} - the archive's "
"integrity is not being checked before it is extracted."
)
return
expected = sidecar.read_text(encoding="utf-8").strip().split()[0:1]
actual = hashlib.sha256(archive.read_bytes()).hexdigest()
if not expected or expected[0].lower() != actual.lower():
fail(
f"{archive.name} does not match {sidecar.name}: expected "
f"{expected[0] if expected else '(unreadable)'}, got {actual}. Re-download the "
"release archive rather than trusting one that failed its own checksum."
)
def _extract_single_top_level_dir(archive: Path, dest: Path) -> Path:
"""Extract `archive` into `dest` and return the one top-level directory it
contained - the shape `.gitea/workflows/release.yml` packs (see its
`Build the distribution tarball` step). Refuses anything else rather than
guessing which part is the machinery."""
with tarfile.open(archive) as tf:
names = [n for n in tf.getnames() if n not in ("", ".")]
top_levels = {n.split("/", 1)[0] for n in names}
if len(top_levels) != 1:
fail(
f"{archive.name} does not have exactly one top-level directory (found "
f"{len(top_levels)}: {', '.join(sorted(top_levels)) or '(empty archive)'}) - this "
"is not the shape a release tarball has, and `dist upgrade` refuses to guess "
"which part is the machinery."
)
return dest # unreachable: fail() raises typer.Exit
try:
tf.extractall(dest, filter="data") # noqa: S202 - trusted local archive, path-checked below
except TypeError:
# Python < 3.12 has no `filter=` argument. Same guard by hand:
# refuse any member whose extracted path would land outside dest.
resolved_dest = dest.resolve()
for member in tf.getmembers():
if not (resolved_dest / member.name).resolve().is_relative_to(resolved_dest):
fail(
f"{archive.name} contains a path that escapes the extraction directory: "
f"{member.name}"
)
return dest # unreachable
tf.extractall(dest) # noqa: S202 - every member path-checked above
return dest / next(iter(top_levels))
@contextmanager
def _resolved_source(source: Path):
"""Yield the directory holding a distribution export: `source` itself if
it already is one, or the single top-level directory of a `.tar.gz`
extracted into a scratch directory that is cleaned up afterward."""
if source.is_dir():
yield source
return
if not source.is_file():
fail(f"{source} does not exist.")
return
_verify_sha256_sidecar(source)
with tempfile.TemporaryDirectory(prefix="wikitool-upgrade-") as tmp:
yield _extract_single_top_level_dir(source, Path(tmp))
def _git_working_tree_status() -> Optional[str]:
"""`git status --porcelain` for `config.ROOT`, or None if it is not a git
repository at all - which is a valid, if unprotected, state for a tarball
instance, not a reason to refuse."""
result = subprocess.run(
["git", "-C", str(config.ROOT), "status", "--porcelain"],
capture_output=True,
text=True,
)
return result.stdout if result.returncode == 0 else None
def _report_plan(
classification: FileClassification,
migration_chain: list["kb_state.Migration"],
boundary_crossing: bool,
local_version: "version_mod.Version",
new_version: "version_mod.Version",
) -> None:
console.print(f"{local_version} -> {new_version}")
if boundary_crossing:
console.print(
f"[bold yellow]Crosses a compatibility boundary[/bold yellow] "
f"({local_version.compat_key} -> {new_version.compat_key}) - this is not a drop-in "
"swap; check the release notes before proceeding."
)
console.print(
f"{len(classification.unchanged)} unchanged, {len(classification.new)} new, "
f"{len(classification.blocked)} locally changed, {len(classification.removed)} removed "
"from the release."
)
if classification.modified:
console.print(f"[bold]Locally modified ({len(classification.modified)}):[/bold]")
for relative in classification.modified:
console.print(f" - {relative}")
if classification.deleted:
console.print(f"[bold]Locally deleted ({len(classification.deleted)}):[/bold]")
for relative in classification.deleted:
console.print(f" - {relative}")
if classification.removed:
console.print("[dim]No longer part of the release, not written or removed by default:[/dim]")
for relative in classification.removed:
console.print(f" [dim]- {relative}[/dim]")
if migration_chain:
console.print(
f"[cyan]{len(migration_chain)} migration(s) will be outstanding after this "
"upgrade, in this order:[/cyan]"
)
for position, migration in enumerate(migration_chain, start=1):
console.print(f" {position}. {migration.target} {migration.name} ({migration.kind})")
console.print("Report only - `dist upgrade` never runs a migration. See `wikitool migrate status`.")
@app.command("upgrade")
def upgrade_command(
source: Path = typer.Argument(
..., help="An extracted distribution directory, or a release .tar.gz archive"
),
dry_run: bool = typer.Option(
False, "--dry-run", help="Classify and report, without writing anything"
),
keep_local: bool = typer.Option(
False, "--keep-local",
help="Proceed even with locally changed files - leave each one untouched rather than aborting",
),
prune: bool = typer.Option(
False, "--prune",
help="Also delete files the new release no longer ships, if they are unchanged since install",
),
allow_pre: bool = typer.Option(
False, "--pre", help="Allow a pre-release (-beta.N) source tree - release.yml never publishes one",
),
):
"""Apply a stack update `dist export` produced - the write half of
`version check`. Never downloads anything: `source` is an already-fetched
export directory or `.tar.gz` archive. Writes exactly the new release
stamp's `files` block, minus what an export re-seeds every time
(`kb/log.md`, `raw/*/.gitkeep`) or seeds once and the instance owns from
then on (`.wikitool-kb.json`, `CHANGES.md`), classifying every candidate
against the *old* stamp's recorded digest: unchanged files are
overwritten silently, new files are created, and a locally modified or
deleted file is never silently overwritten - `dist upgrade` aborts unless
`--keep-local` says to leave it alone. Reports the migration chain the new
machinery would owe without running any of it (there is no `migrate run`).
Refuses on a missing local release stamp, a downgrade, a pre-release
source without `--pre`, or a dirty working tree. Never touches git.
See Gitea #7 and `INSTALL.md` § "Eine Instanz aktualisieren"."""
run_upgrade(
source, dry_run=dry_run, keep_local=keep_local, prune=prune, allow_pre=allow_pre
)
def run_upgrade(
source: Path,
dry_run: bool = False,
keep_local: bool = False,
prune: bool = False,
allow_pre: bool = False,
) -> None:
"""The upgrade itself, free of Typer's option objects - see `run_export`
for why this split exists."""
try:
local_version = version_mod.read_version()
except version_mod.VersionError as exc:
fail(f"{exc} - this tree has no stack version to upgrade from.")
return
old_stamp = version_mod.read_stamp()
if not old_stamp or not isinstance(old_stamp.get("files"), dict):
fail(
f"No local {version_mod.RELEASE_STAMP_FILENAME} (or it carries no `files` block). "
"Without it, `dist upgrade` cannot tell a file this instance edited from one it "
"merely received, and it refuses to guess. A checkout with shared git history takes "
"stack updates via `wikitool upstream merge` instead - it has the same information "
"as a merge base. A tarball instance that has lost its stamp has no repair path "
"today; see Gitea #7 \"Bewusst offen gelassen\"."
)
return
old_files = old_stamp["files"]
try:
kb_version = kb_state.read_kb_version()
except version_mod.VersionError as exc:
fail(str(exc))
return
if kb_version is None:
fail(
f"{kb_state.KB_STATE_FILENAME} is missing - this instance has never declared what "
"shape its content is in. Run `wikitool migrate baseline <version>` before upgrading."
)
return
outstanding = kb_state.chain(kb_state.load_migrations(), kb_version, local_version.base)
if outstanding:
fail(
f"{len(outstanding)} migration(s) are already outstanding against the installed "
f"machinery ({kb_version} -> {local_version}) - `wikitool migrate status` names them. "
"Finish them before upgrading further: a machinery swap on top of an unfinished "
"migration leaves the corpus in a shape no version describes."
)
return
tree_status = _git_working_tree_status()
if tree_status is None:
console.print(
"[yellow]WARN[/yellow] Not a git repository (or git is unavailable) - proceeding "
"without the dirty-tree check a repository would get."
)
elif tree_status.strip():
fail(
"Working tree is not clean (`git status --porcelain` printed something). "
"`dist upgrade` refuses to start on a dirty tree so a refusal never has to guess "
"which changes were already there. Commit or stash first."
)
return
with _resolved_source(source) as new_root:
version_path = new_root / version_mod.VERSION_FILENAME
if not version_path.is_file():
fail(f"{rel_path(new_root)} has no VERSION - not a distribution export.")
return
try:
new_version = version_mod.Version.parse(version_path.read_text(encoding="utf-8"))
except version_mod.VersionError as exc:
fail(str(exc))
return
if new_version.is_prerelease and not allow_pre:
fail(
f"{new_version} is a running candidate (-beta.N). `.gitea/workflows/release.yml` "
"never publishes one, so a candidate tree can only come from a dev checkout by "
"hand - pass --pre if that is deliberate."
)
return
if new_version < local_version:
fail(f"{new_version} is older than the installed {local_version} - refusing a downgrade.")
return
if new_version == local_version:
success(f"Already at {local_version}. Nothing to do.")
return
stamp_path = new_root / version_mod.RELEASE_STAMP_FILENAME
if not stamp_path.is_file():
fail(f"{rel_path(new_root)} has no {version_mod.RELEASE_STAMP_FILENAME} - not a distribution export.")
return
try:
new_stamp = json.loads(stamp_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError) as exc:
fail(f"{version_mod.RELEASE_STAMP_FILENAME} in the source is not readable JSON: {exc}")
return
new_files = new_stamp.get("files") if isinstance(new_stamp, dict) else None
if not isinstance(new_files, dict):
fail(f"{version_mod.RELEASE_STAMP_FILENAME} in the source carries no `files` block.")
return
classification = _classify_files(old_files, new_files)
migration_chain = kb_state.chain(
kb_state.load_migrations(new_root / "instructions" / kb_state.MIGRATIONS_SUBDIR),
kb_version,
new_version.base,
)
boundary_crossing = local_version.compat_key != new_version.compat_key
_report_plan(classification, migration_chain, boundary_crossing, local_version, new_version)
# Dry-run's whole purpose is to preview this classification - including
# the blocked list - without raising, so it must be checked before the
# abort below rather than after: a blocked file must never turn
# `--dry-run` into a non-zero exit, or the flag stops being safe to run
# freely.
if dry_run:
success(f"Dry run: would upgrade {local_version} -> {new_version}. Nothing written.")
return
if classification.blocked and not keep_local:
fail(
f"{len(classification.blocked)} locally changed file(s) (listed above) would be "
"silently overwritten. Pass --keep-local to upgrade anyway and leave every one of "
"them untouched, or reconcile them by hand first. Nothing was written."
)
return
to_write = sorted(classification.unchanged + classification.new)
for relative in to_write:
src = new_root / relative
dst = config.ROOT / relative
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dst)
shutil.copy2(stamp_path, config.ROOT / version_mod.RELEASE_STAMP_FILENAME)
pruned: list[str] = []
if prune:
for relative in classification.removed:
digest = old_files.get(relative)
if digest is None:
continue
status = kb_state.compare_against_stamp({relative: digest}).get(relative)
if status != kb_state.UNCHANGED:
continue
target = config.ROOT / relative
if target.is_file():
target.unlink()
pruned.append(relative)
skipped = classification.blocked if keep_local else []
summary = (
f"Upgraded {local_version} -> {new_version}: {len(to_write)} file(s) written"
+ (f", {len(skipped)} left untouched (--keep-local)" if skipped else "")
+ (f", {len(pruned)} pruned" if pruned else "")
+ "."
)
if migration_chain:
summary += (
f" {len(migration_chain)} migration(s) now outstanding - run `wikitool migrate status`."
)
summary += (
" Nothing was committed. Now run, in order: `wikitool instructions sync`, `doctor`, "
"`docs verify`, `instructions verify`, `lint` - then restart the agent session."
)
success(summary)