torbenandClaude Opus 5.5
dd565d249f
Files changed: - CHANGES.md - VERSION - docs/language-boundaries.md - tools/chemenu/commands/dist_cmd.py - types/type-guidance.md - types/type-spec.md Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
1657 lines
78 KiB
Python
1657 lines
78 KiB
Python
"""`wikitool dist export` - build a distributable, contentless copy of this
|
|
repo's machinery.
|
|
|
|
`export` copies the pipeline's schema/compiler/control-plane layers (types/,
|
|
tools/, instructions/, the stage contracts) into an empty target, with no kb/
|
|
pages, no raw/ content, and no git history - see instructions/setup-instance.md
|
|
for what happens after. It never calls git.
|
|
|
|
The two binding-but-instance-owned documents under kb/ - each collection's
|
|
COLLECTION.md and kb/CONVENTIONS.md - cross as `.template` and are adopted by a
|
|
rename, the same split USER.md/SOUL.md use at the repo root.
|
|
|
|
Three independent exclusion mechanisms feed the plan, for three different
|
|
shapes of "does not belong in someone else's instance":
|
|
|
|
- Every copied text file passes through `strip_markers()`, which removes any
|
|
region between `<!-- dist:strip-start -->` and `<!-- dist:strip-end -->`,
|
|
markers included - for dev-only *content inside* a file that is otherwise
|
|
shipped (e.g. a routing line in AGENTS.md).
|
|
- `instructions/dev/` is pruned from the copy wholesale - for dev-only
|
|
*whole files* (procedures and the skill that switches an agent into
|
|
tool-development mode). One-way: nothing reconstructs it in a distributed
|
|
instance, on purpose - see instructions/dev/ itself for the current
|
|
contents and AGENTS.md's routing line for what a dev instance sees instead.
|
|
- Build output under `tools/` is dropped, by directory (`TOOLS_EXCLUDE_DIRS`)
|
|
where it has one, and by filename (`_is_coverage_output`) where it does not.
|
|
Not dev-only but *derived*: recomputable, and measured against this repo's
|
|
own test run rather than the receiving instance's.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Callable, NamedTuple, Optional, Sequence, Union
|
|
|
|
import typer
|
|
|
|
from chemenu import (
|
|
blocks,
|
|
cli_contract,
|
|
config,
|
|
conventions,
|
|
kb_collections,
|
|
kb_state,
|
|
ownership,
|
|
toc,
|
|
toolpaths,
|
|
version as version_mod,
|
|
)
|
|
from chemenu.commands._util import console, fail, rel_path, success, today_iso
|
|
|
|
app = typer.Typer(help="Build a distributable copy of the wiki machinery.")
|
|
|
|
MARKER_START = "<!-- dist:strip-start -->"
|
|
MARKER_END = "<!-- dist:strip-end -->"
|
|
|
|
DIST_TEMPLATES_DIR = Path(__file__).resolve().parent.parent / "dist_templates"
|
|
|
|
# Root files copied verbatim (after marker-stripping). INSTALL.md is optional
|
|
# here: it does not exist until the distribution docs land, and `export`
|
|
# must not fail just because a later stage of the same repo hasn't shipped
|
|
# yet.
|
|
#
|
|
# The personalization *templates* ship; the filled `USER.md`/`SOUL.md` never
|
|
# do. This allowlist is what makes that split automatic - a file is copied
|
|
# because it is named here, so an instance's own personalization is excluded
|
|
# by construction rather than by a rule someone has to remember.
|
|
#
|
|
# `ENVIRONMENT.md.template` rides the same split for the same reason: a
|
|
# distribution can describe what the file is for, but never what a particular
|
|
# checkout's harness, MCP servers and remotes are. The filled `ENVIRONMENT.md`
|
|
# is additionally gitignored, so it is excluded twice over.
|
|
#
|
|
# `CLAUDE.md` is harness glue, not a second control plane: Claude Code loads it
|
|
# and does not load `AGENTS.md`, so it ships for the same reason
|
|
# `.claude/settings.json` does - a distributed instance running that harness
|
|
# would otherwise start every session without the control plane.
|
|
#
|
|
# `INSTALL-MCP.md` ships beside `INSTALL.md` and for the same reason: the MCP
|
|
# read server is part of what an instance *has*, even though its dependency is
|
|
# optional. A distribution whose server is present but undocumented is one
|
|
# whose operator finds the module by reading the source.
|
|
#
|
|
# `DEVELOPMENT.md` is deliberately **absent** from this tuple, unlike every
|
|
# other root doc above. It documents the release workflow (`version bump` ->
|
|
# `version release` -> `publish` -> CI tags) and points at `instructions/dev/`,
|
|
# which this same function excludes wholesale a few lines down - a distributed
|
|
# instance has no release workflow, no CI and no issue board, so it has
|
|
# nothing for that document to describe. Do not "fix" this by adding it back:
|
|
# a root file absent from ROOT_FILES is silently skipped by every export, and
|
|
# that silence is the correct behaviour here, not a gap.
|
|
ROOT_FILES = (
|
|
"AGENTS.md", "CLAUDE.md", "README.md", "EVALS.md", "INSTALL.md", "INSTALL-MCP.md",
|
|
".gitignore", ".gitattributes", "VERSION",
|
|
*config.LICENSE_FILES,
|
|
*config.PERSONALIZATION_TEMPLATES,
|
|
config.ENVIRONMENT_TEMPLATE,
|
|
)
|
|
|
|
# The one part of ROOT_FILES that may not be quietly skipped. Every other entry
|
|
# copies only `if source.is_file()`, which is right for `INSTALL.md` (it did not
|
|
# exist until the distribution docs landed) and wrong for a licence: an export
|
|
# that silently omits it hands the receiving instance the AGPL-covered `tools/`
|
|
# tree with no licence text, which is a violation the moment that instance is
|
|
# pushed anywhere public. Missing means the export is broken, not minimal.
|
|
REQUIRED_ROOT_FILES = config.LICENSE_FILES
|
|
|
|
# Harness-specific session-tracing config: generic machinery (feeds
|
|
# tools/chemenu/telemetry/ and tools/trace_ingest.py via EVALS.md), not
|
|
# personal state - unlike `.obsidian/`/`.vscode/`, which are never copied.
|
|
HOOK_DIRS = (".github/hooks", ".vibe")
|
|
|
|
# tools/ subpaths never copied - build/venv/cache artifacts, not machinery.
|
|
# `htmlcov/` is coverage.py's HTML report: derived output, and a large tree of
|
|
# it, measured against the source repo's own test run. `.coveragerc` beside it
|
|
# *does* ship, the same way `pytest.ini` does - it is configuration, not output.
|
|
TOOLS_EXCLUDE_DIRS = {".venv", "__pycache__", ".pytest_cache", ".wikitool_session", "htmlcov"}
|
|
|
|
# The rest of coverage's output lands beside the code rather than in a directory
|
|
# of its own - `.coverage`, `coverage.xml`, and `.coverage.<host>.<pid>` under a
|
|
# parallel run - so a directory exclusion cannot reach it. Same argument as
|
|
# `reports/`: derived, recomputable, and about the source repo rather than about
|
|
# the instance that would receive it.
|
|
COVERAGE_OUTPUT_NAMES = frozenset({".coverage", "coverage.xml"})
|
|
|
|
|
|
def _is_coverage_output(filename: str) -> bool:
|
|
return filename in COVERAGE_OUTPUT_NAMES or filename.startswith(".coverage.")
|
|
|
|
|
|
# instructions/dev/ holds stack-development-only procedures and the skill
|
|
# that switches an agent into tool-development mode - never shipped to a
|
|
# distributed instance. One-way: there is no `enable-dev`-style command that
|
|
# reconstructs it afterwards, unlike the marker-block content below.
|
|
INSTRUCTIONS_EXCLUDE_DIRS = {"dev"}
|
|
|
|
# Stage contracts that are not collections and carry no pages: copied as a
|
|
# single file each, nothing else from their directory. `kb/` is excluded here
|
|
# - it is a content stage too, but it has collections underneath it, so its
|
|
# contract is handled by `build_plan` alongside them rather than as a bare
|
|
# stage copy. Derived from `ownership.CONTENT_STAGES` rather than listed
|
|
# again, so this loop and `ownership.is_stack_owned` cannot name a different
|
|
# stage without one of them failing its own test.
|
|
CONTRACT_ONLY_STAGES = tuple(
|
|
f"{stage}/CONTRACT.md" for stage in ownership.CONTENT_STAGES if stage != "kb"
|
|
)
|
|
|
|
# Single tracked files copied out of an otherwise-untouched, partially-ignored
|
|
# directory. `.claude/` holds the harness's own session-tracing config
|
|
# (`settings.json`, tracked) alongside generated skill copies and personal
|
|
# untracked state (`.claude/skills/`, `.claude/settings.local.json`) - neither
|
|
# of which belongs in a distribution. Adding `.claude` to HOOK_DIRS would copy
|
|
# the whole directory, skills included; a single-file entry avoids that
|
|
# without needing an exclude set HOOK_DIRS doesn't otherwise carry.
|
|
SINGLE_FILES = (".claude/settings.json",)
|
|
|
|
Content = Union[str, bytes]
|
|
|
|
|
|
class PlannedFile(NamedTuple):
|
|
content: Content
|
|
executable: bool = False
|
|
|
|
|
|
_MARKER_TOKEN_RE = re.compile(re.escape(MARKER_START) + "|" + re.escape(MARKER_END))
|
|
# The leading/trailing `\n?` consume the blank line on each side of the
|
|
# block - the convention is that a marker block always sits as its own
|
|
# paragraph. Without eating both, a strip leaves two blank lines where the
|
|
# clean file only ever had one.
|
|
_MARKER_BLOCK_RE = re.compile(
|
|
r"\n?" + re.escape(MARKER_START) + r".*?" + re.escape(MARKER_END) + r"\n?", re.DOTALL
|
|
)
|
|
|
|
|
|
def _validate_markers(text: str, label: str) -> None:
|
|
"""A marker file must be a sequence of well-formed, non-nested
|
|
start/end pairs. Malformed markers would make `strip_markers` remove
|
|
either too little or too much, silently - this fails loudly instead."""
|
|
depth = 0
|
|
for match in _MARKER_TOKEN_RE.finditer(text):
|
|
if match.group() == MARKER_START:
|
|
if depth != 0:
|
|
fail(f"{label}: nested dist:strip-start markers are not supported")
|
|
depth = 1
|
|
else:
|
|
if depth != 1:
|
|
fail(f"{label}: dist:strip-end without a matching dist:strip-start")
|
|
depth = 0
|
|
if depth != 0:
|
|
fail(f"{label}: dist:strip-start without a matching dist:strip-end")
|
|
|
|
|
|
def strip_markers(text: str) -> str:
|
|
"""Remove every marked region, markers included. Generic by design: it
|
|
does not matter what is inside, or how many regions a file has."""
|
|
return _MARKER_BLOCK_RE.sub("", text)
|
|
|
|
|
|
def _is_executable(path: Path) -> bool:
|
|
return bool(path.stat().st_mode & stat.S_IXUSR)
|
|
|
|
|
|
def _read_planned_file(path: Path, label: str) -> PlannedFile:
|
|
executable = _is_executable(path)
|
|
try:
|
|
text = path.read_text(encoding="utf-8")
|
|
except UnicodeDecodeError:
|
|
return PlannedFile(path.read_bytes(), executable)
|
|
# Marker syntax is an HTML/Markdown comment convention, scoped to .md
|
|
# files on purpose: applying it to every text file would let the marker
|
|
# strings themselves - inline here as Python string literals - match as
|
|
# a region in this file's own source when tools/ gets copied, and eat
|
|
# the code between them.
|
|
if path.suffix != ".md":
|
|
return PlannedFile(text, executable)
|
|
_validate_markers(text, label)
|
|
stripped = strip_markers(text)
|
|
# A table-of-contents region describes the file it sits in, and stripping
|
|
# just changed that file: `AGENTS.md`'s marked block wraps the whole
|
|
# `## Developing this stack` section, so the shipped copy has one heading
|
|
# fewer than the working tree's TOC lists. Regenerating here - after the
|
|
# strip, on the text that actually ships - is what keeps a fresh
|
|
# instance's own `docs verify` green on the very first run. A file with
|
|
# no region, or one the strip pushed under the threshold, is handled by
|
|
# `toc.upsert` itself.
|
|
if blocks.find(stripped, toc.REGION_NAME) is not None:
|
|
stripped = toc.upsert(stripped)
|
|
return PlannedFile(stripped, executable)
|
|
|
|
|
|
def _copy_tree(
|
|
source_root: Path,
|
|
dest_prefix: str,
|
|
exclude_dirs: frozenset[str],
|
|
exclude_file: Optional[Callable[[str], bool]] = None,
|
|
) -> dict[str, PlannedFile]:
|
|
"""Every file under source_root, marker-stripped, keyed by its
|
|
destination-relative path. Excluded directories are pruned during the
|
|
walk rather than filtered after, so a large `.venv/` is never read.
|
|
`exclude_file` drops individual files by name, for output that lands
|
|
beside the code instead of in a directory a prune could catch."""
|
|
files: dict[str, PlannedFile] = {}
|
|
if not source_root.is_dir():
|
|
return files
|
|
for dirpath, dirnames, filenames in os.walk(source_root):
|
|
dirnames[:] = sorted(d for d in dirnames if d not in exclude_dirs)
|
|
for filename in sorted(filenames):
|
|
if exclude_file is not None and exclude_file(filename):
|
|
continue
|
|
path = Path(dirpath) / filename
|
|
relative = path.relative_to(source_root).as_posix()
|
|
dest_rel = f"{dest_prefix}/{relative}"
|
|
files[dest_rel] = _read_planned_file(path, dest_rel)
|
|
return files
|
|
|
|
|
|
def _digest(content: Content) -> str:
|
|
data = content if isinstance(content, bytes) else content.encode("utf-8")
|
|
return "sha256:" + hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def build_stamp(plan: dict[str, PlannedFile], origin: "Origin") -> str:
|
|
"""The release stamp written into every export.
|
|
|
|
Two jobs. The version and origin fields are what `version check` compares
|
|
against a release feed - without them an instance cannot tell which stack
|
|
it is running. The per-file digests are for the update *after* detection:
|
|
they record what the machinery looked like when it was installed, which is
|
|
the only way a later upgrade can tell a file the instance edited from one
|
|
it merely received. Nothing reads them today; writing them now is what
|
|
keeps that upgrade from needing a format change.
|
|
"""
|
|
stamp = {
|
|
"schema": version_mod.STAMP_SCHEMA,
|
|
"version": str(version_mod.read_version()),
|
|
"exported_at": today_iso(),
|
|
"source_repo": origin.source_repo,
|
|
"source_commit": origin.source_commit,
|
|
"release_url": origin.release_url,
|
|
"update_url": origin.update_url or version_mod.DEFAULT_UPDATE_URL,
|
|
"files": {relative: _digest(planned.content) for relative, planned in sorted(plan.items())},
|
|
}
|
|
return json.dumps(stamp, indent=2, sort_keys=False) + "\n"
|
|
|
|
|
|
class Origin(NamedTuple):
|
|
"""Where this export came from. Supplied by the caller (the release
|
|
workflow knows the commit and the release URL); `dist export` itself never
|
|
calls git, so it cannot discover any of it."""
|
|
|
|
source_repo: Optional[str] = None
|
|
source_commit: Optional[str] = None
|
|
release_url: Optional[str] = None
|
|
update_url: Optional[str] = None
|
|
|
|
|
|
def instance_owned_type_stems() -> set[str]:
|
|
"""Type-spec stems whose instances are knowledge pages, and which therefore
|
|
belong to the instance rather than to the stack.
|
|
|
|
The line is `root:`, and it was already in the frontmatter before anyone
|
|
drew it: `root: kb` means the type describes a page the instance writes, so
|
|
the file is the instance's to change. Anything else - `instruction`
|
|
(`root: repo`), `lint-report` (no `base_dir` at all), `type-spec` itself -
|
|
describes a stack artifact and ships verbatim.
|
|
|
|
Ownership, not language. What such a file is *written in* is decided by who
|
|
reads each half, not by who owns the file: its `## Template` block and its
|
|
`layout:` titles become page text and follow `kb/CONVENTIONS.md`, while the
|
|
authoring guidance around them addresses an agent and stays English like the
|
|
rest of the control plane (AGENTS.md § File naming, types/type-spec.md
|
|
§ Who owns a type-spec). That the two halves share one file, and what it
|
|
costs, is docs/ownership-and-templates.md § Where the file boundary strains.
|
|
|
|
Read from `types/` rather than listed, so an instance adding its own page
|
|
type gets the same treatment without a code change.
|
|
"""
|
|
from chemenu.type_resolver import resolver
|
|
|
|
stems: set[str] = set()
|
|
for type_path, frontmatter in resolver.list_type_specs():
|
|
stem = Path(type_path).stem
|
|
if stem == "type-spec":
|
|
continue
|
|
if not frontmatter.get("base_dir"):
|
|
continue
|
|
if (frontmatter.get("root") or "kb") != "kb":
|
|
continue
|
|
stems.add(stem)
|
|
return stems
|
|
|
|
|
|
# The suffix a type-spec's own two files carry - `<stem>.md` and
|
|
# `<stem>.schema.yaml` - as opposed to a sibling file that merely starts with
|
|
# the same stem, such as `<stem>.guidance.md` (Gitea #104). Checked as an
|
|
# exact suffix rather than by splitting on the first `.`, which is what let
|
|
# `entity.guidance.md` be mistaken for the `entity` type-spec's own file
|
|
# before this existed - a stack-owned file re-keyed as though it were the
|
|
# instance's `.template` to adopt, and flagged as a leak by the other call
|
|
# site for not being one.
|
|
_TYPE_SCHEMA_SUFFIX = ".schema.yaml"
|
|
|
|
|
|
def _owned_type_stem(relative: str) -> Optional[str]:
|
|
"""The type stem `relative` (a path under `types/`, no `.template`
|
|
suffix) names, if it is that type-spec's own `<stem>.md` or
|
|
`<stem>.schema.yaml`, or one of its subtype templates `<stem>.<value>.md`
|
|
(Gitea #117) - `None` for anything else under `types/`, including a
|
|
`<stem>.guidance.md` file. `_plan_types()` and `find_leaks()` both ask this
|
|
instead of computing their own stem, so the two answer the same question
|
|
about the same path (AGENTS.md invariant 8).
|
|
|
|
A subtype template is page material of its type-spec, so it belongs to
|
|
whoever owns that type-spec: re-keyed as `.template` beside a `root: kb`
|
|
one. Missing it here would ship `entity.person.md` verbatim - a file the
|
|
next `dist upgrade` overwrites in an instance that has adopted and
|
|
rewritten it."""
|
|
from chemenu.type_resolver import split_subtype_template_name
|
|
|
|
name = relative.rsplit("/", 1)[-1]
|
|
if name.endswith(_TYPE_SCHEMA_SUFFIX):
|
|
return name[: -len(_TYPE_SCHEMA_SUFFIX)]
|
|
if (split := split_subtype_template_name(name)) is not None:
|
|
return split[0]
|
|
if name.endswith(".md") and "." not in name[: -len(".md")]:
|
|
return name[: -len(".md")]
|
|
return None
|
|
|
|
|
|
def _plan_types() -> dict[str, PlannedFile]:
|
|
"""`types/`, with the page type-specs re-keyed as templates.
|
|
|
|
Same split as the collection contracts, for the same reason and by the same
|
|
mechanism: the shipped content is a working default rather than something
|
|
wrong for the receiver, so the file itself crosses - under a name that has
|
|
to be adopted before it counts. A type-spec's `.schema.yaml` travels with
|
|
it, because the two are one type (see types/type-spec.md § Anatomy) and
|
|
adopting half of it would leave a spec validated by a file it does not own.
|
|
|
|
Its subtype templates `<name>.<value>.md` (Gitea #117) travel the same way,
|
|
each as its own `.template`: page material of the type, adopted or left
|
|
lying independently of the type-spec beside it.
|
|
|
|
A type-spec's optional `<name>.guidance.md` (Gitea #104) is the opposite:
|
|
stack-owned even where the type-spec itself is instance-owned, and ships
|
|
verbatim beside the `.template` - `_owned_type_stem` is what keeps it out
|
|
of this re-keying despite sharing the type-spec's own stem.
|
|
"""
|
|
plan = _copy_tree(config.TYPES_DIR, "types", frozenset())
|
|
stems = instance_owned_type_stems()
|
|
if not stems:
|
|
return plan
|
|
|
|
rekeyed: dict[str, PlannedFile] = {}
|
|
for relative, planned in plan.items():
|
|
stem = _owned_type_stem(relative)
|
|
if stem is not None and stem in stems:
|
|
rekeyed[f"{relative}.template"] = planned
|
|
else:
|
|
rekeyed[relative] = planned
|
|
return rekeyed
|
|
|
|
|
|
def build_plan(origin: Optional[Origin] = None) -> dict[str, PlannedFile]:
|
|
"""Every (destination-relative path -> planned file) the export writes."""
|
|
plan: dict[str, PlannedFile] = {}
|
|
|
|
missing_licences = [
|
|
name for name in REQUIRED_ROOT_FILES if not (config.ROOT / name).is_file()
|
|
]
|
|
if missing_licences:
|
|
fail(
|
|
"export would ship code without its licence: "
|
|
+ ", ".join(missing_licences)
|
|
+ " missing from the source tree. Restore them before exporting - a "
|
|
"distribution carrying tools/ without LICENSE is a copyleft violation "
|
|
"the moment the receiving instance is published."
|
|
)
|
|
|
|
for name in ROOT_FILES:
|
|
source = config.ROOT / name
|
|
if source.is_file():
|
|
plan[name] = _read_planned_file(source, name)
|
|
|
|
plan.update(_copy_tree(config.INSTRUCTIONS_DIR, "instructions", frozenset(INSTRUCTIONS_EXCLUDE_DIRS)))
|
|
plan.update(_plan_types())
|
|
plan.update(_copy_tree(
|
|
config.ROOT / "tools", "tools", frozenset(TOOLS_EXCLUDE_DIRS), _is_coverage_output
|
|
))
|
|
for hook_dir in HOOK_DIRS:
|
|
plan.update(_copy_tree(config.ROOT / hook_dir, hook_dir, frozenset()))
|
|
|
|
# docs/ is stack background - why the stack is built the way it is - and
|
|
# ships verbatim like instructions/ and types/: it carries no page, no
|
|
# frontmatter, and (AGENTS.md § File naming) no normative sentence, so
|
|
# there is nothing instance-owned in it to split off as a .template.
|
|
plan.update(_copy_tree(config.ROOT / "docs", "docs", frozenset()))
|
|
|
|
# `kb/CONTRACT.md` is stack-owned and ships verbatim; everything beside it
|
|
# under `kb/` is the instance's own and ships only as a `.template`. That is
|
|
# the personalization split (`USER.md`/`SOUL.md`) one directory down, and
|
|
# the reason for it is the same: a distribution can say what the file
|
|
# decides, never what this instance decided.
|
|
kb_contract = config.KB_DIR / "CONTRACT.md"
|
|
if kb_contract.is_file():
|
|
plan["kb/CONTRACT.md"] = _read_planned_file(kb_contract, "kb/CONTRACT.md")
|
|
|
|
conventions_template = config.KB_DIR / conventions.CONVENTIONS_TEMPLATE
|
|
if conventions_template.is_file():
|
|
rel = f"kb/{conventions.CONVENTIONS_TEMPLATE}"
|
|
plan[rel] = _read_planned_file(conventions_template, rel)
|
|
|
|
# A collection contract is instance-owned too, but unlike `USER.md` the
|
|
# shipped content is not *wrong* for the receiver - it is the profile this
|
|
# repo's own collections adopted, and a fine starting point. So the file
|
|
# itself ships, under the template name: one source of truth here, and a
|
|
# receiving instance that has to rename it before it counts. Keeping a
|
|
# separate `.template` beside each contract would have meant maintaining two
|
|
# near-identical copies of the same text, which is the drift AGENTS.md
|
|
# invariant 8 exists to prevent.
|
|
for collection in kb_collections.iter_kb_collections():
|
|
source = collection / kb_collections.CONTRACT_NAME
|
|
rel = f"kb/{collection.name}/{kb_collections.CONTRACT_NAME}.template"
|
|
plan[rel] = _read_planned_file(source, rel)
|
|
|
|
for relative in CONTRACT_ONLY_STAGES:
|
|
source = config.ROOT / relative
|
|
if source.is_file():
|
|
plan[relative] = _read_planned_file(source, relative)
|
|
|
|
for relative in SINGLE_FILES:
|
|
source = config.ROOT / relative
|
|
if source.is_file():
|
|
plan[relative] = _read_planned_file(source, relative)
|
|
|
|
# `raw/` and `incoming/` are both flat (Gitea #67 removed type
|
|
# subdirectories from the addressing scheme entirely - a file's location
|
|
# under `raw/` is a date shard computed by `raw accept`, never a hand-picked
|
|
# type), so an export creates no subdirectories under either root. The
|
|
# exported `.gitignore` excludes what lands in `incoming/` but not its
|
|
# `.gitkeep` (`/incoming/*` plus `!/incoming/.gitkeep`, Gitea #88), so the
|
|
# anchor is tracked and a plain clone of the instance has the directory
|
|
# without any bootstrap step re-creating it.
|
|
plan["raw/.gitkeep"] = PlannedFile("")
|
|
plan["incoming/.gitkeep"] = PlannedFile("")
|
|
|
|
plan["kb/log.md"] = PlannedFile((DIST_TEMPLATES_DIR / "log.md").read_text(encoding="utf-8"))
|
|
plan["CHANGES.md"] = PlannedFile((DIST_TEMPLATES_DIR / "CHANGES.md").read_text(encoding="utf-8"))
|
|
|
|
# A fresh instance's content is empty, so it is trivially in the shape this
|
|
# machinery expects - which is exactly what makes the initial declaration
|
|
# safe to write here rather than leaving it to `migrate baseline`. Only an
|
|
# instance predating this file has to answer that question by hand.
|
|
#
|
|
# `.base`, not the raw `VERSION`: a content shape has no beta channel
|
|
# (`kb_state.read_kb_version` refuses one), so exporting mid-candidate
|
|
# still declares the release the content is shaped for, not the candidate
|
|
# in progress. The stamp below carries the honest, suffix-inclusive value -
|
|
# the two files answer different questions.
|
|
plan[kb_state.KB_STATE_FILENAME] = PlannedFile(
|
|
kb_state.render_kb_state(version_mod.read_version().base, [])
|
|
)
|
|
|
|
# Last, so it can digest everything above it. It is the one file in the
|
|
# export that describes the export rather than being copied into it.
|
|
plan[version_mod.RELEASE_STAMP_FILENAME] = PlannedFile(
|
|
build_stamp(plan, origin or Origin())
|
|
)
|
|
|
|
return plan
|
|
|
|
|
|
# Content that must never appear in a plan, expressed structurally rather than
|
|
# by matching text. Three allowlists feed `build_plan`, and each one holds only
|
|
# because someone remembered the rule when they edited it - nothing re-checks
|
|
# the result. This does.
|
|
#
|
|
# The checks are deliberately structural: a filled personalization file, a kb
|
|
# page, a raw source, a dev-only instruction. A text-pattern scan (hostnames,
|
|
# IP literals) was considered and rejected - the project's own host legitimately
|
|
# appears in INSTALL.md and version.py, so such a scan would either whitelist
|
|
# the very string it is looking for or cry wolf on every export.
|
|
#
|
|
# `COLLECTION.md` and `CONVENTIONS.md` are deliberately *not* allowed through
|
|
# any more. Both bind, and both are the instance's to write, so they cross the
|
|
# boundary as `.template` and are adopted by a rename - a plan carrying the
|
|
# filled name would hand a new instance this one's authoring conventions as
|
|
# though they were the stack's.
|
|
#
|
|
# What counts as machinery under kb/ or raw/ is no longer a second list here:
|
|
# it is `ownership.is_stack_owned`. Only the export-only stubs
|
|
# (`ownership.EXPORT_STUB_NAMES`) are allowed here without also being
|
|
# stack-owned - export writes a fresh one rather than shipping this repo's.
|
|
_CONTENT_PREFIXES = ("kb/", "raw/")
|
|
_INSTANCE_OWNED_KB_FILES = (kb_collections.CONTRACT_NAME, conventions.CONVENTIONS_FILENAME)
|
|
|
|
|
|
def find_leaks(plan: dict[str, PlannedFile]) -> list[str]:
|
|
"""Planned paths that carry one instance's own data instead of machinery."""
|
|
owned_types = instance_owned_type_stems()
|
|
leaks: list[str] = []
|
|
for relative in sorted(plan):
|
|
name = relative.rsplit("/", 1)[-1]
|
|
if name in config.PERSONALIZATION_FILES or name == config.ENVIRONMENT_FILE:
|
|
leaks.append(f"{relative} (one instance's own personalization)")
|
|
elif relative.startswith("kb/") and name in _INSTANCE_OWNED_KB_FILES:
|
|
leaks.append(f"{relative} (this instance's authoring conventions; ship the .template)")
|
|
elif (
|
|
relative.startswith("types/")
|
|
and not relative.endswith(".template")
|
|
and (owned_stem := _owned_type_stem(relative)) is not None
|
|
and owned_stem in owned_types
|
|
):
|
|
leaks.append(
|
|
f"{relative} (this instance's page type-spec or subtype template; "
|
|
"ship the .template)"
|
|
)
|
|
elif relative.startswith("instructions/dev/"):
|
|
leaks.append(f"{relative} (stack-development only)")
|
|
elif (
|
|
relative.startswith(_CONTENT_PREFIXES)
|
|
and not ownership.is_stack_owned(relative)
|
|
and not ownership.is_export_stub(name)
|
|
):
|
|
leaks.append(f"{relative} (wiki content, not machinery)")
|
|
return leaks
|
|
|
|
|
|
def _write_plan(target: Path, plan: dict[str, PlannedFile]) -> None:
|
|
for relative, planned in plan.items():
|
|
dest = target / relative
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
if isinstance(planned.content, bytes):
|
|
dest.write_bytes(planned.content)
|
|
else:
|
|
dest.write_text(planned.content, encoding="utf-8", newline="\n")
|
|
if planned.executable:
|
|
dest.chmod(dest.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
|
|
|
|
|
@cli_contract.record(cli_contract.CommandRecord(
|
|
path="dist export",
|
|
summary="Write a contentless, distributable copy of this repo's machinery.",
|
|
synopsis=(cli_contract.Variant(
|
|
usage="dist export <target> [--dry-run] [--source-repo U] [--source-commit SHA] "
|
|
"[--release-url U] [--update-url U]",
|
|
),),
|
|
properties=cli_contract.Properties(
|
|
effect=cli_contract.Effect.WRITE,
|
|
idempotent=cli_contract.Idempotent.YES,
|
|
atomic="Yes - nothing is written until every file is planned",
|
|
budget=cli_contract.Budget.COUNTED,
|
|
),
|
|
notes=(
|
|
"Writes a contentless, distributable copy of this repo's machinery into an empty or "
|
|
"new `<target>` directory.",
|
|
"Ships `AGENTS.md`/`README.md`/`EVALS.md` with any `dist:strip-start`...`dist:strip-end` "
|
|
"marker region removed, `instructions/` (minus `instructions/dev/`), `types/` (the "
|
|
"`root: kb` page type-specs, their schemas and their subtype templates "
|
|
"`types/<type>.<value>.md` re-keyed as `.template`, the stack's own verbatim), `docs/` verbatim, `tools/` (no venv/caches), the "
|
|
"`.github/hooks/`+`.vibe/` session-tracing config plus `.claude/settings.json`, "
|
|
"`kb/CONTRACT.md` (no pages, no areas) and `VERSION`.",
|
|
"Ships `raw/` and `incoming/` as flat roots, each with a `.gitkeep` and no "
|
|
"subdirectories. `incoming/.gitkeep` is trackable and survives becoming a git "
|
|
"repository, so a plain clone gets the directory without any bootstrap step.",
|
|
"Ships templates, never the filled files: `USER.md.template`/`SOUL.md.template`, "
|
|
"`kb/CONVENTIONS.md.template`, and each collection's contract re-keyed as "
|
|
"`kb/<name>/COLLECTION.md.template`. The filled "
|
|
"`USER.md`/`SOUL.md`/`kb/CONVENTIONS.md`/`kb/<name>/COLLECTION.md`/"
|
|
"`types/<page-type>.md`/`types/<page-type>.<value>.md` bind their instance; `find_leaks` "
|
|
"refuses a plan carrying one.",
|
|
"Writes a generated `.wikitool-release.json` stamp: version, export date, origin, and "
|
|
"a sha256 per exported file - the base a later upgrade compares against.",
|
|
"The four origin options only fill stamp fields: `export` never calls git and cannot "
|
|
"discover them.",
|
|
"A build and test tool: every release is an export packed as a tarball, and an instance "
|
|
"is installed from such a release, never from an export directly.",
|
|
"One-way: no command reconstructs a distributed instance into a dev instance - work on "
|
|
"the stack in a clone of the origin repo.",
|
|
"`--dry-run` lists every file it would write, and writes nothing.",
|
|
),
|
|
failures=(
|
|
cli_contract.Failure(
|
|
cause="`<target>` exists and is not empty, or is not a directory",
|
|
reaction="Point `<target>` at an empty (or new) directory and retry",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="The tree has no readable `VERSION`",
|
|
reaction="Fix `VERSION`, then retry",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="A licence file is missing from the source tree",
|
|
reaction="Restore it, then retry",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="The plan would carry this instance's own data (a filled personalization, "
|
|
"conventions or page type-spec file)",
|
|
reaction="Report it: it is an allowlist bug in `dist export`, not something to "
|
|
"work around by deleting files from the target",
|
|
),
|
|
),
|
|
examples=(
|
|
"tools/wikitool dist export ../my-wiki --dry-run",
|
|
"tools/wikitool dist export ../my-wiki",
|
|
),
|
|
never=(
|
|
"Never merge an export into a non-empty directory by hand.",
|
|
),
|
|
see_also=(
|
|
"`instructions/setup-instance.md` - installs a release, which is this export as a tarball",
|
|
"`wikitool dist upgrade` - applies a later export to an existing instance",
|
|
"`wikitool version show` - reads the stamp this writes",
|
|
),
|
|
))
|
|
@app.command("export")
|
|
def export_command(
|
|
target: Path = typer.Argument(
|
|
..., help="Directory to write the distribution into. Must not exist, or must be empty."
|
|
),
|
|
dry_run: bool = typer.Option(
|
|
False, "--dry-run", help="List what would be written, without writing anything."
|
|
),
|
|
source_repo: Optional[str] = typer.Option(
|
|
None, "--source-repo", help="Repository this export was built from (recorded in the stamp)"
|
|
),
|
|
source_commit: Optional[str] = typer.Option(
|
|
None, "--source-commit", help="Commit this export was built from (recorded in the stamp)"
|
|
),
|
|
release_url: Optional[str] = typer.Option(
|
|
None, "--release-url", help="Release page this export ships as (recorded in the stamp)"
|
|
),
|
|
update_url: Optional[str] = typer.Option(
|
|
None, "--update-url", help="Release feed `version check` should ask (recorded in the stamp)"
|
|
),
|
|
):
|
|
"""Export a contentless, distributable copy of this repo's machinery:
|
|
AGENTS.md/README.md (dev-instance-only marker blocks removed),
|
|
instructions/ (no instructions/dev/), types/, docs/ verbatim,
|
|
tools/ (no venv/caches),
|
|
the .github/hooks/+.vibe session-tracing config plus .claude/settings.json,
|
|
kb/CONTRACT.md plus a COLLECTION.md.template per collection and
|
|
kb/CONVENTIONS.md.template (no pages, no areas), the flat
|
|
raw/.gitkeep and incoming/.gitkeep anchors, VERSION, the USER.md/SOUL.md
|
|
personalization templates (never the filled files), and a
|
|
.wikitool-release.json stamp. The --source-*/--release-url/--update-url
|
|
options only fill fields in that stamp: `export` never calls git and cannot
|
|
discover them. See instructions/setup-instance.md for what comes next."""
|
|
run_export(
|
|
target,
|
|
dry_run=dry_run,
|
|
origin=Origin(
|
|
source_repo=source_repo,
|
|
source_commit=source_commit,
|
|
release_url=release_url,
|
|
update_url=update_url,
|
|
),
|
|
)
|
|
|
|
|
|
def run_export(target: Path, dry_run: bool = False, origin: Optional[Origin] = None) -> None:
|
|
"""The export itself, free of Typer's option objects so it can be called
|
|
directly - by the command above, and by the tests."""
|
|
target = target.resolve()
|
|
if target.exists():
|
|
if not target.is_dir():
|
|
fail(f"{target} exists and is not a directory.")
|
|
if any(target.iterdir()):
|
|
fail(f"{target} is not empty. `dist export` refuses to write into a non-empty directory.")
|
|
|
|
try:
|
|
plan = build_plan(origin)
|
|
except version_mod.VersionError as exc:
|
|
fail(f"{exc} - a distribution must carry the version it ships.")
|
|
return
|
|
|
|
leaks = find_leaks(plan)
|
|
if leaks:
|
|
fail(
|
|
"export would carry this instance's own data, not just machinery:\n "
|
|
+ "\n ".join(leaks)
|
|
+ "\nThis is an allowlist bug in dist_cmd.py, not something to work "
|
|
"around - fix the allowlist rather than deleting files from the target."
|
|
)
|
|
return
|
|
|
|
if dry_run:
|
|
for relative in sorted(plan):
|
|
typer.echo(f"write {relative}")
|
|
success(f"Dry run: would write {len(plan)} file(s) to {target}. Nothing written.")
|
|
return
|
|
|
|
_write_plan(target, plan)
|
|
success(f"Exported {len(plan)} file(s) to {rel_path(target)}.")
|
|
|
|
|
|
# --- dist adopt --------------------------------------------------------------
|
|
#
|
|
# The other half of the `.template` split above: an instance takes the shipped
|
|
# default as its own by copying it to the unsuffixed name. Only the templates
|
|
# whose shipped text is a working default are in scope - each collection's
|
|
# contract and the page type-specs with their schemas and subtype templates.
|
|
# `kb/CONVENTIONS.md` and the personalization files ship as templates too, but carry a sentinel and
|
|
# exist to be filled in, so a verbatim copy of them would only be a file
|
|
# `doctor` refuses; the agent writes those itself.
|
|
|
|
|
|
def adoptable_templates() -> list[Path]:
|
|
"""Every template `dist adopt` copies when it is given no path, sorted."""
|
|
found = [
|
|
path
|
|
for path in config.KB_DIR.glob(f"*/{kb_collections.CONTRACT_NAME}{toc.TEMPLATE_SUFFIX}")
|
|
if path.is_file()
|
|
]
|
|
found += [
|
|
path for path in config.TYPES_DIR.glob(f"*{toc.TEMPLATE_SUFFIX}") if path.is_file()
|
|
]
|
|
return sorted(found)
|
|
|
|
|
|
@cli_contract.record(cli_contract.CommandRecord(
|
|
path="dist adopt",
|
|
summary="Take shipped templates as this instance's own: copy each to its unsuffixed name.",
|
|
synopsis=(cli_contract.Variant(usage="dist adopt [<template>...] [--dry-run]"),),
|
|
properties=cli_contract.Properties(
|
|
effect=cli_contract.Effect.WRITE,
|
|
idempotent=cli_contract.Idempotent.YES,
|
|
atomic="No - files are copied one by one; a re-run completes an interrupted one",
|
|
budget=cli_contract.Budget.COUNTED,
|
|
),
|
|
notes=(
|
|
"Copies `<name>.template` to `<name>` byte for byte; the template stays where it is, as "
|
|
"the base the next `dist upgrade` compares against.",
|
|
"Without a path: every `kb/<collection>/COLLECTION.md.template` and every "
|
|
"`types/*.template` (the `root: kb` page type-specs, their schemas and their subtype "
|
|
"templates).",
|
|
"With paths: exactly those templates, each of which has to be one of the set above.",
|
|
"Never overwrites: a target that already exists is reported as kept and left untouched.",
|
|
"Not for `kb/CONVENTIONS.md.template` or the personalization templates - those carry a "
|
|
"sentinel and are filled in, not copied.",
|
|
"`--dry-run` lists what it would copy and keep, and writes nothing.",
|
|
),
|
|
failures=(
|
|
cli_contract.Failure(
|
|
cause="A named path does not exist, or is not a collection contract or page type-spec "
|
|
"template",
|
|
reaction="Not transient - name a template from the set in NOTES, or call it without "
|
|
"a path",
|
|
),
|
|
),
|
|
examples=(
|
|
"tools/wikitool dist adopt",
|
|
"tools/wikitool dist adopt types/project.md.template types/project.schema.yaml.template",
|
|
"tools/wikitool dist adopt --dry-run",
|
|
),
|
|
never=(
|
|
"Never delete a target to make `dist adopt` replace it - a filled file is the "
|
|
"instance's own work.",
|
|
),
|
|
see_also=(
|
|
"`instructions/setup-instance.md` - adopts every template on a fresh instance",
|
|
"`instructions/upgrade-instance.md` - adopts a template a release added",
|
|
"`wikitool dist export` - re-keys these files as `.template` in the first place",
|
|
),
|
|
))
|
|
@app.command("adopt")
|
|
def adopt_command(
|
|
templates: Optional[list[Path]] = typer.Argument(
|
|
None, help="Templates to adopt. Default: every collection contract and page type-spec template."
|
|
),
|
|
dry_run: bool = typer.Option(
|
|
False, "--dry-run", help="List what would be copied, without writing anything."
|
|
),
|
|
):
|
|
"""Copy each shipped template to its unsuffixed name, never overwriting a file
|
|
that already exists."""
|
|
run_adopt(templates or [], dry_run=dry_run)
|
|
|
|
|
|
def run_adopt(templates: Sequence[Path], dry_run: bool = False) -> None:
|
|
adoptable = {path.resolve() for path in adoptable_templates()}
|
|
if templates:
|
|
chosen = []
|
|
for given in templates:
|
|
path = given if given.is_absolute() else config.ROOT / given
|
|
if path.resolve() not in adoptable:
|
|
fail(
|
|
f"{given.as_posix()} is not a template `dist adopt` copies - it takes "
|
|
"`kb/<collection>/COLLECTION.md.template` and `types/*.template` only."
|
|
)
|
|
return
|
|
chosen.append(path)
|
|
else:
|
|
chosen = sorted(adoptable)
|
|
|
|
adopted = kept = 0
|
|
for template in chosen:
|
|
target = template.with_name(template.name[: -len(toc.TEMPLATE_SUFFIX)])
|
|
if target.exists():
|
|
typer.echo(f"keep {rel_path(target)} (exists)")
|
|
kept += 1
|
|
continue
|
|
typer.echo(f"adopt {rel_path(template)} -> {rel_path(target)}")
|
|
if not dry_run:
|
|
shutil.copyfile(template, target)
|
|
adopted += 1
|
|
|
|
if dry_run:
|
|
success(f"Dry run: would adopt {adopted} template(s), keep {kept}. Nothing written.")
|
|
else:
|
|
success(f"Adopted {adopted} template(s), kept {kept}.")
|
|
|
|
|
|
# --- dist upgrade ------------------------------------------------------------
|
|
#
|
|
# Apply a release `dist export` produced, rather than merely detecting one
|
|
# (`version check`). Gitea #7 has the full design; the short version: the
|
|
# write set is exactly the *new* stamp's `files` block, minus the paths an
|
|
# export seeds once and the instance owns from then on
|
|
# (`ownership.is_export_stub`, `ownership.is_upgrade_preserved`), plus the
|
|
# stamp itself. Every candidate path is classified against the *old* stamp's
|
|
# recorded digest - unchanged, locally modified, or locally deleted - and a
|
|
# modified/deleted file is never silently overwritten: the run aborts unless
|
|
# `--keep-local` keeps it or `--take-release <path>` names it, which is the
|
|
# difference between a file the instance means to carry and one that drifted.
|
|
# The classification never touches the network: the tree or archive it works
|
|
# on is either supplied by the caller or fetched by `--latest` beforehand.
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class FileClassification:
|
|
"""The four-way split of every path `dist upgrade` would touch, plus the
|
|
fifth direction (`removed`) that has no write set of its own."""
|
|
|
|
unchanged: list[str]
|
|
modified: list[str]
|
|
deleted: list[str]
|
|
new: list[str]
|
|
removed: list[str]
|
|
|
|
@property
|
|
def blocked(self) -> list[str]:
|
|
"""Locally changed paths - modified or deleted - which are never
|
|
silently overwritten."""
|
|
return sorted(self.modified + self.deleted)
|
|
|
|
|
|
def _write_candidates(new_files: dict) -> set[str]:
|
|
"""Every path `dist upgrade` may write, from the new stamp's `files`
|
|
block: everything except the paths an export re-seeds from a blank
|
|
template every time (`ownership.is_export_stub`) and the paths an export
|
|
seeds once and the instance owns afterward (`ownership.is_upgrade_preserved`).
|
|
The release stamp itself is added separately - it is never a member of its
|
|
own `files` block, see `build_stamp`."""
|
|
return {
|
|
relative
|
|
for relative in new_files
|
|
if not ownership.is_export_stub(Path(relative).name)
|
|
and not ownership.is_upgrade_preserved(relative)
|
|
}
|
|
|
|
|
|
def _classify_files(old_files: dict, new_files: dict) -> FileClassification:
|
|
candidates = _write_candidates(new_files)
|
|
recorded_for_candidates = {r: d for r, d in old_files.items() if r in candidates}
|
|
statuses = kb_state.compare_against_stamp(recorded_for_candidates)
|
|
|
|
unchanged: list[str] = []
|
|
modified: list[str] = []
|
|
deleted: list[str] = []
|
|
new: list[str] = []
|
|
for relative in sorted(candidates):
|
|
if relative not in old_files:
|
|
new.append(relative)
|
|
continue
|
|
status = statuses[relative]
|
|
if status == kb_state.UNCHANGED:
|
|
unchanged.append(relative)
|
|
elif status == kb_state.MODIFIED:
|
|
modified.append(relative)
|
|
else:
|
|
deleted.append(relative)
|
|
|
|
removed = sorted(set(old_files) - set(new_files))
|
|
return FileClassification(unchanged, modified, deleted, new, removed)
|
|
|
|
|
|
def _verify_sha256_sidecar(archive: Path) -> None:
|
|
"""WARN, never fail, on a missing sidecar - only a corrupted one that
|
|
*is* present is a reason to stop, per Gitea #7's design table."""
|
|
sidecar = archive.with_name(archive.name + ".sha256")
|
|
if not sidecar.is_file():
|
|
console.print(
|
|
f"[yellow]WARN[/yellow] No {sidecar.name} beside {archive.name} - the archive's "
|
|
"integrity is not being checked before it is extracted."
|
|
)
|
|
return
|
|
expected = sidecar.read_text(encoding="utf-8").strip().split()[0:1]
|
|
actual = hashlib.sha256(archive.read_bytes()).hexdigest()
|
|
if not expected or expected[0].lower() != actual.lower():
|
|
fail(
|
|
f"{archive.name} does not match {sidecar.name}: expected "
|
|
f"{expected[0] if expected else '(unreadable)'}, got {actual}. Re-download the "
|
|
"release archive rather than trusting one that failed its own checksum."
|
|
)
|
|
|
|
|
|
def _extract_single_top_level_dir(archive: Path, dest: Path) -> Path:
|
|
"""Extract `archive` into `dest` and return the one top-level directory it
|
|
contained - the shape `.gitea/workflows/release.yml` packs (see its
|
|
`Build the distribution tarball` step). Refuses anything else rather than
|
|
guessing which part is the machinery."""
|
|
with tarfile.open(archive) as tf:
|
|
names = [n for n in tf.getnames() if n not in ("", ".")]
|
|
top_levels = {n.split("/", 1)[0] for n in names}
|
|
if len(top_levels) != 1:
|
|
fail(
|
|
f"{archive.name} does not have exactly one top-level directory (found "
|
|
f"{len(top_levels)}: {', '.join(sorted(top_levels)) or '(empty archive)'}) - this "
|
|
"is not the shape a release tarball has, and `dist upgrade` refuses to guess "
|
|
"which part is the machinery."
|
|
)
|
|
return dest # unreachable: fail() raises typer.Exit
|
|
try:
|
|
tf.extractall(dest, filter="data") # noqa: S202 - trusted local archive, path-checked below
|
|
except TypeError:
|
|
# Python < 3.12 has no `filter=` argument. Same guard by hand:
|
|
# refuse any member whose extracted path would land outside dest.
|
|
resolved_dest = dest.resolve()
|
|
for member in tf.getmembers():
|
|
if not (resolved_dest / member.name).resolve().is_relative_to(resolved_dest):
|
|
fail(
|
|
f"{archive.name} contains a path that escapes the extraction directory: "
|
|
f"{member.name}"
|
|
)
|
|
return dest # unreachable
|
|
tf.extractall(dest) # noqa: S202 - every member path-checked above
|
|
return dest / next(iter(top_levels))
|
|
|
|
|
|
@contextmanager
|
|
def _resolved_source(source: Path):
|
|
"""Yield the directory holding a distribution export: `source` itself if
|
|
it already is one, or the single top-level directory of a `.tar.gz`
|
|
extracted into a scratch directory that is cleaned up afterward."""
|
|
if source.is_dir():
|
|
yield source
|
|
return
|
|
if not source.is_file():
|
|
fail(f"{source} does not exist.")
|
|
return
|
|
_verify_sha256_sidecar(source)
|
|
with tempfile.TemporaryDirectory(prefix="wikitool-upgrade-") as tmp:
|
|
yield _extract_single_top_level_dir(source, Path(tmp))
|
|
|
|
|
|
def _version_gate(
|
|
new_version: "version_mod.Version",
|
|
local_version: "version_mod.Version",
|
|
allow_pre: bool,
|
|
origin: str,
|
|
) -> bool:
|
|
"""The three version refusals shared by a source tree and the feed: a
|
|
candidate without `--pre`, a downgrade, an equal version. `True` means go
|
|
on; `False` means the run ended successfully as a no-op. `origin` names
|
|
what reported the version, for the message."""
|
|
if new_version.is_prerelease and not allow_pre:
|
|
fail(
|
|
f"{origin} reports {new_version}, a running candidate (-beta.N). "
|
|
"`.gitea/workflows/release.yml` never publishes one, so it can only come from a "
|
|
"dev checkout or a hand-made release - pass --pre if that is deliberate."
|
|
)
|
|
if new_version < local_version:
|
|
fail(f"{origin} reports {new_version}, older than the installed {local_version} - refusing a downgrade.")
|
|
if new_version == local_version:
|
|
success(f"Already at {local_version}. Nothing to do.")
|
|
return False
|
|
return True
|
|
|
|
|
|
def _query_feed(
|
|
stamp: dict,
|
|
local_version: "version_mod.Version",
|
|
allow_pre: bool,
|
|
expect: Optional["version_mod.Version"],
|
|
) -> Optional[tuple["version_mod.LatestRelease", str, Optional[str]]]:
|
|
"""Ask the release feed which release is latest and decide, before any
|
|
download, whether there is anything to fetch. Returns the release with the
|
|
feed URL and token, or `None` when the installed version is already it."""
|
|
url = version_mod.update_url(stamp)
|
|
token = os.environ.get(version_mod.UPDATE_TOKEN_ENV, "").strip() or None
|
|
console.print(f"Asking the release feed {url}", highlight=False)
|
|
try:
|
|
release = version_mod.fetch_latest_assets(url, token)
|
|
except version_mod.VersionError as exc:
|
|
fail(str(exc))
|
|
if expect is not None and release.version != expect:
|
|
fail(
|
|
f"The feed's latest release is {release.version}, not the expected {expect} - "
|
|
"nothing was downloaded. The feed moved on since the notes were read, or "
|
|
f"the wrong version was named: read `wikitool version notes` for {release.version} "
|
|
"before deciding to go there instead."
|
|
)
|
|
if not _version_gate(release.version, local_version, allow_pre, "the release feed"):
|
|
return None
|
|
return release, url, token
|
|
|
|
|
|
@contextmanager
|
|
def _downloaded_source(release: "version_mod.LatestRelease", feed_url: str, token: Optional[str]):
|
|
"""Yield the directory of the release the feed announced: download its
|
|
archive and checksum into a scratch directory, hold the archive to the
|
|
checksum (a missing checksum is an error, never a WARN), and unpack it.
|
|
A release without both assets is refused before the first download. The
|
|
scratch directory is removed on every exit, `--dry-run` included."""
|
|
archive_url, sum_url = _asset_urls(release)
|
|
with tempfile.TemporaryDirectory(prefix="wikitool-upgrade-") as tmp:
|
|
archive = Path(tmp) / release.archive_name
|
|
checksum = archive.with_name(archive.name + version_mod.CHECKSUM_SUFFIX)
|
|
for url, dest in ((archive_url, archive), (sum_url, checksum)):
|
|
console.print(f"Downloading {url}", highlight=False)
|
|
try:
|
|
version_mod.download_asset(url, dest, version_mod.token_for_asset(feed_url, url, token))
|
|
except version_mod.VersionError as exc:
|
|
fail(str(exc))
|
|
_verify_sha256_sidecar(archive)
|
|
yield _extract_single_top_level_dir(archive, Path(tmp) / "unpacked")
|
|
|
|
|
|
def _asset_urls(release: "version_mod.LatestRelease") -> tuple[str, str]:
|
|
try:
|
|
return release.asset_urls()
|
|
except version_mod.VersionError as exc:
|
|
fail(str(exc))
|
|
raise # unreachable: fail() raises typer.Exit
|
|
|
|
|
|
def _git_working_tree_status() -> Optional[str]:
|
|
"""`git status --porcelain` for `config.ROOT`, or None if it is not a git
|
|
repository at all - which is a valid, if unprotected, state for a tarball
|
|
instance, not a reason to refuse."""
|
|
result = subprocess.run(
|
|
[toolpaths.git(), "-C", str(config.ROOT), "status", "--porcelain"],
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
)
|
|
return result.stdout if result.returncode == 0 else None
|
|
|
|
|
|
def _resolve_take_release(
|
|
take_release: Optional[Sequence[str]], classification: FileClassification
|
|
) -> set[str]:
|
|
"""The blocked paths `--take-release` names, refusing any that is not
|
|
actually blocked.
|
|
|
|
A path that silently does nothing is the worse answer: the operator asked
|
|
for a local change to be discarded and would be told the upgrade went
|
|
fine, having kept it. Checked before `--dry-run` returns, so a typo
|
|
surfaces in the preview rather than in the writing run."""
|
|
if not take_release:
|
|
return set()
|
|
blocked = set(classification.blocked)
|
|
wanted = {path.strip() for path in take_release if path.strip()}
|
|
unknown = sorted(wanted - blocked)
|
|
if unknown:
|
|
listed = "\n".join(f" - {path}" for path in classification.blocked) or " (none)"
|
|
fail(
|
|
f"--take-release names {len(unknown)} path(s) that are not locally changed: "
|
|
f"{', '.join(unknown)}. Only a path this run reports as locally modified or "
|
|
f"locally deleted can be taken from the release. Reported as locally changed:\n"
|
|
f"{listed}"
|
|
)
|
|
return set() # unreachable: fail() raises typer.Exit
|
|
return wanted
|
|
|
|
|
|
def _refusal_for_blocked(
|
|
target: str, undecided: list[str], classification: FileClassification
|
|
) -> str:
|
|
"""The abort text for blocked paths no flag has answered for.
|
|
|
|
It spells all three answers out with a ready-to-paste command line -
|
|
the same shape the Mass-Update Gate uses for its `--confirm` line -
|
|
because the one thing a reader must not take away is that any of them is
|
|
the default. A run on a real instance read the old wording, which named
|
|
only `--keep-local` and "reconcile by hand", as "the default takes the
|
|
release's version" and called the command with no flag at all."""
|
|
paths = " ".join(undecided)
|
|
kept_again = (
|
|
"they are reported again on every future upgrade"
|
|
if len(classification.blocked) > 1
|
|
else "it is reported again on every future upgrade"
|
|
)
|
|
return (
|
|
f"{len(undecided)} locally changed file(s) (listed above) would be silently "
|
|
f"overwritten. Nothing was written, and none of these three is the default:\n"
|
|
f" - take the release's version and discard the local change:\n"
|
|
f" dist upgrade {target} --take-release {paths}\n"
|
|
f" - keep every local change and upgrade around them ({kept_again}):\n"
|
|
f" dist upgrade {target} --keep-local\n"
|
|
f" - reconcile them by hand first, then re-run."
|
|
)
|
|
|
|
|
|
def _report_plan(
|
|
classification: FileClassification,
|
|
migration_chain: list["kb_state.Migration"],
|
|
boundary_crossing: bool,
|
|
local_version: "version_mod.Version",
|
|
new_version: "version_mod.Version",
|
|
taken: set[str] = frozenset(),
|
|
) -> None:
|
|
console.print(f"{local_version} -> {new_version}")
|
|
if boundary_crossing:
|
|
console.print(
|
|
f"[bold yellow]Crosses a compatibility boundary[/bold yellow] "
|
|
f"({local_version.compat_key} -> {new_version.compat_key}) - this is not a drop-in "
|
|
"swap; check the release notes before proceeding."
|
|
)
|
|
console.print(
|
|
f"{len(classification.unchanged)} unchanged, {len(classification.new)} new, "
|
|
f"{len(classification.blocked)} locally changed, {len(classification.removed)} removed "
|
|
"from the release."
|
|
)
|
|
|
|
def _mark(relative: str) -> str:
|
|
return " [cyan](--take-release: overwritten from the release)[/cyan]" if relative in taken else ""
|
|
|
|
if classification.modified:
|
|
console.print(f"[bold]Locally modified ({len(classification.modified)}):[/bold]")
|
|
for relative in classification.modified:
|
|
console.print(f" - {relative}{_mark(relative)}")
|
|
if classification.deleted:
|
|
console.print(f"[bold]Locally deleted ({len(classification.deleted)}):[/bold]")
|
|
for relative in classification.deleted:
|
|
console.print(f" - {relative}{_mark(relative)}")
|
|
if classification.removed:
|
|
console.print("[dim]No longer part of the release, not written or removed by default:[/dim]")
|
|
for relative in classification.removed:
|
|
console.print(f" [dim]- {relative}[/dim]")
|
|
if migration_chain:
|
|
console.print(
|
|
f"[cyan]{len(migration_chain)} migration(s) will be outstanding after this "
|
|
"upgrade, in this order:[/cyan]"
|
|
)
|
|
for position, migration in enumerate(migration_chain, start=1):
|
|
console.print(f" {position}. {migration.target} {migration.name} ({migration.kind})")
|
|
console.print("Report only - `dist upgrade` never runs a migration. See `wikitool migrate status`.")
|
|
|
|
|
|
@cli_contract.record(cli_contract.CommandRecord(
|
|
path="dist upgrade",
|
|
summary="Apply a stack update `dist export` produced - the write half of `version check`.",
|
|
synopsis=(cli_contract.Variant(
|
|
usage="dist upgrade (<source> | --latest [--expect <version>]) [--dry-run] "
|
|
"[--keep-local] [--take-release <path>]... [--prune] [--pre]",
|
|
),),
|
|
properties=cli_contract.Properties(
|
|
effect=cli_contract.Effect.WRITE,
|
|
idempotent=cli_contract.Idempotent.NO,
|
|
atomic="Yes for every refusal - nothing is written. Once writing starts it is a plain "
|
|
"sequential file copy with no partial-state cleanup: an interruption mid-copy "
|
|
"(killed process, disk full) can leave the tree part-old, part-new",
|
|
budget=cli_contract.Budget.COUNTED,
|
|
network=cli_contract.Network.YES,
|
|
),
|
|
notes=(
|
|
"Exactly one of `<source>` and `--latest` names the release. `<source>` is an "
|
|
"already-fetched export directory or `.tar.gz` release archive and touches no network: "
|
|
"the archive is verified against a sibling `.sha256` if one is present (a missing one "
|
|
"is a WARN, not a block) and must unpack to exactly one top-level directory - the "
|
|
"shape `.gitea/workflows/release.yml` packs.",
|
|
"`--latest` asks the release feed (`update_url` of the local stamp, overridable with "
|
|
"`$WIKITOOL_UPDATE_URL`; `$WIKITOOL_UPDATE_TOKEN` is sent along) which release is "
|
|
"latest, checks that version - `--expect`, a downgrade, a pre-release without `--pre`, "
|
|
"already installed - before any download, then downloads the archive and its "
|
|
"`.sha256` into a scratch directory removed on every exit. The checksum is mandatory "
|
|
"here (a release without one, or an archive that fails it, is an error, not a WARN) "
|
|
"and the archive's own `VERSION` must equal the feed's version. The asset URLs are the "
|
|
"feed's own `browser_download_url` values, and the token reaches an asset download only "
|
|
"if it is on the feed's origin. The checksum protects against transfer errors, not "
|
|
"against a feed that is itself compromised - authenticity is the trust in the feed's "
|
|
"host.",
|
|
"`--expect <version>` (only with `--latest`) pins the release the feed may announce: a "
|
|
"different latest version is refused before anything is downloaded. Pass the version "
|
|
"`wikitool version notes` was read for, in the dry run and in the real run alike.",
|
|
"The write set is exactly the *new* `.wikitool-release.json`'s `files` block, minus "
|
|
"what an export re-seeds from a blank template every time (`kb/log.md`, "
|
|
"`raw/.gitkeep`) or seeds once and the instance owns from then on "
|
|
"(`.wikitool-kb.json`, `CHANGES.md`), plus the stamp itself, always rewritten.",
|
|
"Every candidate path is classified against the *local* `.wikitool-release.json`'s "
|
|
"recorded digest: unchanged is overwritten silently, absent from the old stamp is "
|
|
"created, and locally modified or locally deleted is **never** silently overwritten.",
|
|
"Locally changed files abort the run with the full list; the abort text names the "
|
|
"three answers with the command line filled in, and none of them is the default.",
|
|
"`--keep-local` proceeds and leaves every locally changed file untouched. The new "
|
|
"stamp is still written whole, recording the release's digest for files that were "
|
|
"not written, so a skipped file keeps diverging and is reported again on every later "
|
|
"run.",
|
|
"`--take-release <path>` (repeatable) writes the release's version over the named "
|
|
"path, discarding the local change, and re-creates it if it was locally deleted. The "
|
|
"path then matches the stamp and stops being reported.",
|
|
"The two are decided per path and compose on one call: without `--keep-local`, a "
|
|
"locally changed path that no `--take-release` names still aborts the run.",
|
|
"A path in the old stamp but not the new one is reported as no longer part of the "
|
|
"release and left alone, unless `--prune` is passed, which removes it only if it is "
|
|
"still unchanged since installation.",
|
|
"Reports the migration chain the new machinery would owe, but never runs any of it - "
|
|
"there is no `migrate run`.",
|
|
"Reports, but does not block on, a crossed compatibility boundary.",
|
|
"The local preconditions - `VERSION`, the release stamp, `.wikitool-kb.json`, no "
|
|
"outstanding migration, a clean working tree - are checked before the source is read. "
|
|
"Not being a git repository at all is a WARN, not a refusal.",
|
|
"Never touches git - no commit, no push.",
|
|
"`--dry-run` classifies and reports without writing; a pre-release (`-beta.N`) source "
|
|
"needs `--pre`. With `--latest` it still downloads and verifies the archive - that is "
|
|
"the only way to classify - and removes it again.",
|
|
"An interrupted write is not resumed automatically: compare the tree against the "
|
|
"printed classification and finish or revert by hand.",
|
|
"The closing report names `instructions/upgrade-instance.md`, which carries the order "
|
|
"for everything after the swap and resumes at `instructions sync`.",
|
|
),
|
|
failures=(
|
|
cli_contract.Failure(
|
|
cause="The source's version equals the installed one - a no-op success",
|
|
reaction="",
|
|
code=0,
|
|
),
|
|
cli_contract.Failure(
|
|
cause="Both `<source>` and `--latest`, or neither; or `--expect` without `--latest`",
|
|
reaction="Not transient - name exactly one source, and pass `--expect` only with "
|
|
"`--latest`",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="Local `VERSION` missing, or no local `.wikitool-release.json` with a `files` "
|
|
"block",
|
|
reaction="Not transient - fix the named precondition and retry. A clone of the "
|
|
"origin repo is a development checkout and takes no `dist upgrade` at all",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="`.wikitool-kb.json` is missing",
|
|
reaction="Run `wikitool migrate baseline <version>`, then retry",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="A migration is already outstanding against the *installed* machinery",
|
|
reaction="Finish it first - `wikitool migrate status` names it - then retry",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="The working tree is dirty",
|
|
reaction="Commit or stash first, then retry",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="`<source>` does not exist, fails its `.sha256`, or does not unpack to exactly "
|
|
"one top-level directory",
|
|
reaction="Fix the path or re-download the release archive, then retry",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="`--latest`: the release feed cannot be reached, or answers with something "
|
|
"that is not a release",
|
|
reaction="Transient - retry once; then report the URL from the message, or take the "
|
|
"release page's archive by hand and pass it as `<source>`",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="`--latest --expect`: the feed's latest release is another version than the "
|
|
"one expected; nothing was downloaded",
|
|
reaction="Not transient - read `wikitool version notes` for the version the message "
|
|
"names, then either expect that one or stop",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="`--latest`: the release publishes no archive or no `.sha256` under the "
|
|
"expected name; nothing was downloaded",
|
|
reaction="Not transient - the message lists the assets present and the release page; "
|
|
"report it there rather than upgrading without the checksum",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="`--latest`: an asset download fails, or the archive fails its `.sha256`",
|
|
reaction="Retry once; if it fails again, report the exact message - do not fall back "
|
|
"to an unchecked archive",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="`--latest`: the downloaded archive's `VERSION` is not the version the feed "
|
|
"announced",
|
|
reaction="Not transient - an inconsistent release; report it against the release page",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="The source carries no `VERSION`, `.wikitool-release.json` or `files` block",
|
|
reaction="Point `<source>` at a distribution export, then retry",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="The source's version is older than the installed one, or a pre-release "
|
|
"without `--pre`",
|
|
reaction="Not transient - choose another source, or pass `--pre` for a pre-release",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="A `--take-release` path this run does not classify as locally changed - the "
|
|
"one refusal a `--dry-run` also raises",
|
|
reaction="Correct it against the locally-changed list the refusal prints, then retry",
|
|
),
|
|
cli_contract.Failure(
|
|
cause="Locally changed files that neither `--keep-local` nor a `--take-release` "
|
|
"answers for; nothing was written",
|
|
reaction="Choose one of the three answers the refusal names, re-run lines filled in: "
|
|
"`--take-release <path>` to write the release's version over it (ends the "
|
|
"divergence), `--keep-local` to leave them untouched (reported again on every "
|
|
"later run until they stop diverging), or reconcile by hand and retry",
|
|
),
|
|
),
|
|
examples=(
|
|
"tools/wikitool dist upgrade ../chemenu-7.1.0.tar.gz --dry-run",
|
|
"tools/wikitool dist upgrade ../chemenu-7.1.0.tar.gz",
|
|
"tools/wikitool dist upgrade ../chemenu-7.1.0.tar.gz --take-release tools/README.md",
|
|
"tools/wikitool dist upgrade --latest --expect 8.0.0 --dry-run",
|
|
),
|
|
never=(
|
|
"Never treat any of the three answers to locally changed files as the default.",
|
|
),
|
|
see_also=(
|
|
"`wikitool version check` - finds out whether an update exists",
|
|
"`wikitool version notes` - the notes of the release `--expect` should name",
|
|
"`instructions/upgrade-instance.md` - the order after the swap",
|
|
"`INSTALL.md` § \"Version und Updates\" - which release, whether to take it, where "
|
|
"the tarball comes from",
|
|
"`wikitool migrate status` - the migrations the report names",
|
|
),
|
|
))
|
|
@app.command("upgrade")
|
|
def upgrade_command(
|
|
source: Optional[Path] = typer.Argument(
|
|
None, help="An extracted distribution directory, or a release .tar.gz archive - or use --latest"
|
|
),
|
|
latest: bool = typer.Option(
|
|
False, "--latest",
|
|
help="Download the release the feed announces as latest instead of naming a <source>",
|
|
),
|
|
expect: Optional[str] = typer.Option(
|
|
None, "--expect",
|
|
help="With --latest: refuse, before downloading anything, unless the feed's latest release "
|
|
"is this version (the one `version notes` was read for)",
|
|
),
|
|
dry_run: bool = typer.Option(
|
|
False, "--dry-run", help="Classify and report, without writing anything"
|
|
),
|
|
keep_local: bool = typer.Option(
|
|
False, "--keep-local",
|
|
help="Proceed even with locally changed files - leave each one untouched rather than aborting",
|
|
),
|
|
take_release: list[str] = typer.Option(
|
|
None, "--take-release",
|
|
help="Overwrite this locally changed path with the release's version, discarding the local "
|
|
"change. Repeatable, and each path must be one this run reports as locally changed. The "
|
|
"counterpart to --keep-local, which keeps the change and reports it again on every future "
|
|
"upgrade",
|
|
),
|
|
prune: bool = typer.Option(
|
|
False, "--prune",
|
|
help="Also delete files the new release no longer ships, if they are unchanged since install",
|
|
),
|
|
allow_pre: bool = typer.Option(
|
|
False, "--pre", help="Allow a pre-release (-beta.N) source tree - release.yml never publishes one",
|
|
),
|
|
):
|
|
"""Apply a stack update `dist export` produced - the write half of `version check`.
|
|
\f
|
|
`source` is an already-fetched export directory or `.tar.gz` archive;
|
|
`--latest` instead downloads the archive and its checksum from the
|
|
release feed (`--expect <version>` pins which release that may be) and
|
|
holds it to the checksum before anything else. Writes exactly the new release
|
|
stamp's `files` block, minus what an export re-seeds every time
|
|
(`kb/log.md`, `raw/.gitkeep`) or seeds once and the instance owns from
|
|
then on (`.wikitool-kb.json`, `CHANGES.md`), classifying every candidate
|
|
against the *old* stamp's recorded digest: unchanged files are
|
|
overwritten silently, new files are created, and a locally modified or
|
|
deleted file is never silently overwritten - `dist upgrade` aborts unless
|
|
`--keep-local` says to leave it alone or `--take-release <path>` names it
|
|
as one to overwrite from the release. Reports the migration chain the new
|
|
machinery would owe without running any of it (there is no `migrate run`).
|
|
Refuses on a missing local release stamp, a downgrade, a pre-release
|
|
source without `--pre`, a dirty working tree, or a `--take-release` path
|
|
that is not locally changed. Never touches git.
|
|
See Gitea #7 and `INSTALL.md` § "Eine Instanz aktualisieren"."""
|
|
run_upgrade(
|
|
source,
|
|
dry_run=dry_run,
|
|
keep_local=keep_local,
|
|
take_release=take_release,
|
|
prune=prune,
|
|
allow_pre=allow_pre,
|
|
latest=latest,
|
|
expect=expect,
|
|
)
|
|
|
|
|
|
def run_upgrade(
|
|
source: Optional[Path] = None,
|
|
dry_run: bool = False,
|
|
keep_local: bool = False,
|
|
take_release: Optional[Sequence[str]] = None,
|
|
prune: bool = False,
|
|
allow_pre: bool = False,
|
|
latest: bool = False,
|
|
expect: Optional[str] = None,
|
|
) -> None:
|
|
"""The upgrade itself, free of Typer's option objects - see `run_export`
|
|
for why this split exists."""
|
|
if latest == (source is not None):
|
|
fail(
|
|
"Name exactly one source: `<source>` (an export directory or a release .tar.gz "
|
|
"already on disk) or `--latest` (the release the feed announces). "
|
|
+ ("Both were given." if latest else "Neither was given.")
|
|
)
|
|
return
|
|
expected_version = None
|
|
if expect is not None:
|
|
if not latest:
|
|
fail("--expect only makes sense with --latest: it pins the release the feed may announce.")
|
|
return
|
|
try:
|
|
expected_version = version_mod.Version.parse(expect)
|
|
except version_mod.VersionError as exc:
|
|
fail(f"--expect: {exc}")
|
|
return
|
|
|
|
try:
|
|
local_version = version_mod.read_version()
|
|
except version_mod.VersionError as exc:
|
|
fail(f"{exc} - this tree has no stack version to upgrade from.")
|
|
return
|
|
|
|
old_stamp = version_mod.read_stamp()
|
|
if not old_stamp or not isinstance(old_stamp.get("files"), dict):
|
|
fail(
|
|
f"No local {version_mod.RELEASE_STAMP_FILENAME} (or it carries no `files` block). "
|
|
"Without it, `dist upgrade` cannot tell a file this instance edited from one it "
|
|
"merely received, and it refuses to guess. A clone of the origin repo is a "
|
|
"development checkout and takes no `dist upgrade` at all. A release instance that "
|
|
"has lost its stamp has no repair path today; see Gitea #7 \"Bewusst offen gelassen\"."
|
|
)
|
|
return
|
|
old_files = old_stamp["files"]
|
|
|
|
try:
|
|
kb_version = kb_state.read_kb_version()
|
|
except version_mod.VersionError as exc:
|
|
fail(str(exc))
|
|
return
|
|
if kb_version is None:
|
|
fail(
|
|
f"{kb_state.KB_STATE_FILENAME} is missing - this instance has never declared what "
|
|
"shape its content is in. Run `wikitool migrate baseline <version>` before upgrading."
|
|
)
|
|
return
|
|
|
|
outstanding = kb_state.chain(kb_state.load_migrations(), kb_version, local_version.base)
|
|
if outstanding:
|
|
fail(
|
|
f"{len(outstanding)} migration(s) are already outstanding against the installed "
|
|
f"machinery ({kb_version} -> {local_version}) - `wikitool migrate status` names them. "
|
|
"Finish them before upgrading further: a machinery swap on top of an unfinished "
|
|
"migration leaves the corpus in a shape no version describes."
|
|
)
|
|
return
|
|
|
|
tree_status = _git_working_tree_status()
|
|
if tree_status is None:
|
|
console.print(
|
|
"[yellow]WARN[/yellow] Not a git repository (or git is unavailable) - proceeding "
|
|
"without the dirty-tree check a repository would get."
|
|
)
|
|
elif tree_status.strip():
|
|
fail(
|
|
"Working tree is not clean (`git status --porcelain` printed something). "
|
|
"`dist upgrade` refuses to start on a dirty tree so a refusal never has to guess "
|
|
"which changes were already there. Commit or stash first."
|
|
)
|
|
return
|
|
|
|
feed = None
|
|
if latest:
|
|
feed = _query_feed(old_stamp, local_version, allow_pre, expected_version)
|
|
if feed is None:
|
|
return
|
|
release, feed_url, feed_token = feed
|
|
source_cm = _downloaded_source(release, feed_url, feed_token)
|
|
target = f"--latest --expect {release.version}"
|
|
else:
|
|
source_cm = _resolved_source(source)
|
|
target = rel_path(source)
|
|
|
|
with source_cm as new_root:
|
|
version_path = new_root / version_mod.VERSION_FILENAME
|
|
if not version_path.is_file():
|
|
fail(f"{rel_path(new_root)} has no VERSION - not a distribution export.")
|
|
return
|
|
try:
|
|
new_version = version_mod.Version.parse(version_path.read_text(encoding="utf-8"))
|
|
except version_mod.VersionError as exc:
|
|
fail(str(exc))
|
|
return
|
|
|
|
if feed is not None and new_version != feed[0].version:
|
|
fail(
|
|
f"The downloaded archive's VERSION says {new_version}, but the feed announced "
|
|
f"{feed[0].version} (tag {feed[0].tag}) - this release is inconsistent, so "
|
|
"nothing was applied. Report it against the release page"
|
|
+ (f" {feed[0].html_url}" if feed[0].html_url else "")
|
|
+ "."
|
|
)
|
|
return
|
|
if not _version_gate(new_version, local_version, allow_pre, "the source"):
|
|
return
|
|
|
|
stamp_path = new_root / version_mod.RELEASE_STAMP_FILENAME
|
|
if not stamp_path.is_file():
|
|
fail(f"{rel_path(new_root)} has no {version_mod.RELEASE_STAMP_FILENAME} - not a distribution export.")
|
|
return
|
|
try:
|
|
new_stamp = json.loads(stamp_path.read_text(encoding="utf-8"))
|
|
except (json.JSONDecodeError, OSError) as exc:
|
|
fail(f"{version_mod.RELEASE_STAMP_FILENAME} in the source is not readable JSON: {exc}")
|
|
return
|
|
new_files = new_stamp.get("files") if isinstance(new_stamp, dict) else None
|
|
if not isinstance(new_files, dict):
|
|
fail(f"{version_mod.RELEASE_STAMP_FILENAME} in the source carries no `files` block.")
|
|
return
|
|
|
|
classification = _classify_files(old_files, new_files)
|
|
migration_chain = kb_state.chain(
|
|
kb_state.load_migrations(new_root / "instructions" / kb_state.MIGRATIONS_SUBDIR),
|
|
kb_version,
|
|
new_version.base,
|
|
)
|
|
boundary_crossing = local_version.compat_key != new_version.compat_key
|
|
|
|
taken = _resolve_take_release(take_release, classification)
|
|
_report_plan(
|
|
classification, migration_chain, boundary_crossing, local_version, new_version, taken
|
|
)
|
|
|
|
# Dry-run's whole purpose is to preview this classification - including
|
|
# the blocked list - without raising, so it must be checked before the
|
|
# abort below rather than after: a blocked file must never turn
|
|
# `--dry-run` into a non-zero exit, or the flag stops being safe to run
|
|
# freely. A bad `--take-release` path is the other way round: it is a
|
|
# mistake in the *argument*, not a state of the tree, so it is resolved
|
|
# above this line and does exit non-zero here - catching a typo in the
|
|
# preview is the whole point of previewing.
|
|
if dry_run:
|
|
success(f"Dry run: would upgrade {local_version} -> {new_version}. Nothing written.")
|
|
return
|
|
|
|
undecided = [path for path in classification.blocked if path not in taken]
|
|
if undecided and not keep_local:
|
|
fail(_refusal_for_blocked(target, undecided, classification))
|
|
return
|
|
|
|
to_write = sorted(classification.unchanged + classification.new + sorted(taken))
|
|
for relative in to_write:
|
|
src = new_root / relative
|
|
dst = config.ROOT / relative
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(src, dst)
|
|
# The new stamp is written whole even after `--keep-local` skipped files: it is the
|
|
# baseline for the next comparison, not a literal inventory of what is on disk. That
|
|
# is what keeps a skipped file diverging - and therefore reported - on every later
|
|
# run, rather than quietly reading as current once it has been skipped once. A
|
|
# `--take-release` path is the opposite case: it was written, so it matches the
|
|
# digest recorded here and stops being reported at all.
|
|
shutil.copy2(stamp_path, config.ROOT / version_mod.RELEASE_STAMP_FILENAME)
|
|
|
|
pruned: list[str] = []
|
|
if prune:
|
|
for relative in classification.removed:
|
|
digest = old_files.get(relative)
|
|
if digest is None:
|
|
continue
|
|
status = kb_state.compare_against_stamp({relative: digest}).get(relative)
|
|
if status != kb_state.UNCHANGED:
|
|
continue
|
|
target = config.ROOT / relative
|
|
if target.is_file():
|
|
target.unlink()
|
|
pruned.append(relative)
|
|
|
|
skipped = undecided if keep_local else []
|
|
summary = (
|
|
f"Upgraded {local_version} -> {new_version}: {len(to_write)} file(s) written"
|
|
+ (f", {len(taken)} taken from the release (--take-release)" if taken else "")
|
|
+ (f", {len(skipped)} left untouched (--keep-local)" if skipped else "")
|
|
+ (f", {len(pruned)} pruned" if pruned else "")
|
|
+ "."
|
|
)
|
|
if migration_chain:
|
|
summary += (
|
|
f" {len(migration_chain)} migration(s) now outstanding - run `wikitool migrate status`."
|
|
)
|
|
# One pointer rather than a second copy of the order: the steps after the
|
|
# swap live in instructions/upgrade-instance.md, which ships with every
|
|
# instance. Naming the resume *command* rather than a step number keeps this
|
|
# line correct when that file's numbering moves.
|
|
summary += (
|
|
" Nothing was committed and nothing is verified yet."
|
|
" `instructions/upgrade-instance.md` carries the order for everything that follows"
|
|
" and resumes with the preflight (`tools/preflight.sh`, or `tools/preflight.ps1` under PowerShell 7), which `tools/wikitool` now"
|
|
" refuses to run without."
|
|
)
|
|
success(summary)
|