From 90ea0eea1bac153e20b8dafaf4db8ff80c228a46 Mon Sep 17 00:00:00 2001 From: Torben Nehmer Date: Wed, 7 Oct 2026 23:10:19 +0200 Subject: [PATCH] feat(workflow): Gitea-Wiki als generierter Spiegel von docs/ (#192) Konverter .gitea/scripts/wiki_mirror.py, Push-Skript wiki-push.sh, Job wiki-mirror in release.yml, Dispatch-Workflow wiki-mirror.yml und --check-Schritt in ci.yml. Version 9.0.0-beta.13. Files changed: - .gitea/scripts/wiki-push.sh - .gitea/scripts/wiki_mirror.py - .gitea/workflows/ci.yml - .gitea/workflows/release.yml - .gitea/workflows/wiki-mirror.yml - CHANGES.md - README.md - VERSION - docs/dev/development.md - tools/chemenu/tests/test_wiki_mirror.py Co-Authored-By: Claude Code Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2 --- .gitea/scripts/wiki-push.sh | 82 +++++++ .gitea/scripts/wiki_mirror.py | 293 ++++++++++++++++++++++ .gitea/workflows/ci.yml | 8 + .gitea/workflows/release.yml | 51 ++++ .gitea/workflows/wiki-mirror.yml | 78 ++++++ CHANGES.md | 3 +- README.md | 5 + VERSION | 2 +- docs/dev/development.md | 37 ++- tools/chemenu/tests/test_wiki_mirror.py | 313 ++++++++++++++++++++++++ 10 files changed, 869 insertions(+), 3 deletions(-) create mode 100755 .gitea/scripts/wiki-push.sh create mode 100755 .gitea/scripts/wiki_mirror.py create mode 100644 .gitea/workflows/wiki-mirror.yml create mode 100644 tools/chemenu/tests/test_wiki_mirror.py diff --git a/.gitea/scripts/wiki-push.sh b/.gitea/scripts/wiki-push.sh new file mode 100755 index 0000000..494e514 --- /dev/null +++ b/.gitea/scripts/wiki-push.sh @@ -0,0 +1,82 @@ +#!/bin/sh +# Make the wiki repository of this repo hold exactly the converter output. +# +# wiki-push.sh +# +# Environment: +# WIKI_TOKEN token with write access to the wiki (required) +# GITHUB_SERVER_URL set by the runner; GITHUB_REPOSITORY likewise +# WIKI_REMOTE overrides the wiki clone URL (the tests point it at a local repository) +# +# The wiki is a mirror, never a source: every file in it is replaced by the converter output, so +# a page the converter no longer produces disappears. Afterwards the working tree minus .git/ and +# the output directory are compared, and any difference fails the script before anything is +# pushed. An unchanged result ends with exit 0 and no commit. +set -eu + +out="${1:?usage: wiki-push.sh }" +version="${2:?usage: wiki-push.sh }" + +if [ -z "${WIKI_TOKEN:-}" ]; then + echo "ERROR WIKI_TOKEN is empty: the wiki mirror needs a token that may push to the wiki repository." >&2 + echo " The job token (gitea.token) is tried first; if Gitea refuses it, the operator adds a" >&2 + echo " repository secret with write access and the workflow passes that one instead." >&2 + exit 1 +fi +[ -f "$out/Home.md" ] || { echo "ERROR $out/Home.md is missing - not a converter output" >&2; exit 1; } + +server="${GITHUB_SERVER_URL:-}" +repo="${GITHUB_REPOSITORY:-}" +remote="${WIKI_REMOTE:-${server}/${repo}.wiki.git}" +api="${server}/api/v1/repos/${repo}" + +# The token travels in a header, not in the URL, so it appears in no git message or process list. +auth="Authorization: Basic $(printf 'wiki-mirror:%s' "$WIKI_TOKEN" | base64 | tr -d '\n')" +git_remote() { git -c "http.extraheader=${auth}" "$@"; } + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +wiki="$work/wiki" + +if ! git_remote clone -q "$remote" "$wiki" 2>"$work/clone.err"; then + echo "Cloning ${remote} failed:" + cat "$work/clone.err" + # A wiki that has never had a page has no repository yet; the first page created through + # the API brings it into existence. + echo "Creating the first page through the API to bring the wiki repository into existence." + content="$(base64 < "$out/Home.md" | tr -d '\n')" + jq -n --arg title Home --arg content "$content" --arg message "Mirror docs at v${version}" \ + '{title: $title, content_base64: $content, message: $message}' > "$work/first-page.json" + status="$(curl -sS -o "$work/first-page.out" -w '%{http_code}' -X POST "${api}/wiki/new" \ + -H "Authorization: token ${WIKI_TOKEN}" -H 'Content-Type: application/json' \ + --data-binary @"$work/first-page.json")" + case "$status" in + 2??) ;; + *) + echo "ERROR the API refused the first page (HTTP ${status}):" >&2 + cat "$work/first-page.out" >&2 + exit 1 + ;; + esac + git_remote clone -q "$remote" "$wiki" +fi + +find "$wiki" -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + +cp -R "$out"/. "$wiki"/ +git -C "$wiki" add -A + +if ! diff -r -x .git "$out" "$wiki" > "$work/diff.txt"; then + echo "ERROR the wiki working tree differs from the converter output after copying:" >&2 + cat "$work/diff.txt" >&2 + exit 1 +fi + +if git -C "$wiki" diff --cached --quiet; then + echo "Wiki already matches v${version}" + exit 0 +fi + +git -C "$wiki" -c user.name='Wiki Mirror' -c user.email='wiki-mirror@noreply.invalid' \ + commit -q -m "Mirror docs at v${version}" +git_remote -C "$wiki" push -q origin HEAD +echo "Wiki now mirrors v${version}" diff --git a/.gitea/scripts/wiki_mirror.py b/.gitea/scripts/wiki_mirror.py new file mode 100755 index 0000000..8666009 --- /dev/null +++ b/.gitea/scripts/wiki_mirror.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Convert the documentation of one release tree into the flat page set of a Gitea wiki. + + wiki_mirror.py --source --out --version [--repo-url ] [--check] + +The output directory becomes the whole content of the wiki repository, so the conversion is +deterministic: the same tree and version give the same bytes. `--check` converts in memory, +writes nothing and fails only on a name collision or an unresolvable relative link. + +Plain standard library on purpose: it runs in CI before any tool environment exists, and it +must work against the tree of any past tag. +""" +from __future__ import annotations + +import argparse +import os +import posixpath +import re +import sys +from urllib.parse import quote, unquote + +DEFAULT_REPO_URL = "https://gitea.nehmer.net/torben/chemenu" +RESERVED_NAMES = {"home", "install", "_sidebar"} +AREA_ORDER = ("tutorials", "how-to", "reference", "explanation") +DEV_AREA = "dev" +FLAT_GROUP = "Documentation" + +_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:") +_FENCE = re.compile(r"^\s*(`{3,}|~{3,})") +_LINK = re.compile(r"""\]\(\s*([^)\s<][^)\s]*)((?:\s+(?:"[^"]*"|'[^']*'))?)\s*\)""") +_H1 = re.compile(r"^# +(.+?)\s*#*\s*$") + + +class Page: + def __init__(self, path: str, name: str, segs: tuple[str, ...] | None, index: bool): + self.path = path + self.name = name + self.segs = segs # sidebar position below docs/; None for Home and Install + self.index = index # a README.md: stands for its directory + + +def _capitalise(segment: str) -> str: + return segment[:1].upper() + segment[1:] + + +def collect_pages(source: str) -> tuple[list[Page], list[str]]: + """Every page to mirror, in a stable order, plus the collisions found among their names.""" + pages: list[Page] = [] + if os.path.isfile(os.path.join(source, "README.md")): + pages.append(Page("README.md", "Home", None, False)) + if os.path.isfile(os.path.join(source, "INSTALL.md")): + pages.append(Page("INSTALL.md", "Install", None, False)) + + docs = os.path.join(source, "docs") + found: list[str] = [] + for base, dirs, files in os.walk(docs): + dirs.sort() + for file in sorted(files): + if file.endswith(".md"): + rel = os.path.relpath(os.path.join(base, file), source) + found.append(rel.replace(os.sep, "/")) + for path in sorted(found): + parts = path[len("docs/"):-len(".md")].split("/") + index = parts[-1] == "README" + if index: + parts = parts[:-1] + name = "-".join(_capitalise(p) for p in parts) if parts else FLAT_GROUP + pages.append(Page(path, name, tuple(parts), index)) + + errors: list[str] = [] + seen: dict[str, str] = {} + for page in pages: + key = page.name.lower() + if key in RESERVED_NAMES and page.segs is not None: + errors.append(f"{page.path}: page name '{page.name}' is reserved for a generated page") + elif key in seen: + errors.append( + f"name collision on '{page.name}' (compared without case): {seen[key]} and {page.path}" + ) + else: + seen[key] = page.path + return pages, errors + + +def mask_code_spans(line: str) -> str: + """The line with every inline code span blanked to NULs, same length, so offsets still match.""" + out = list(line) + n = len(line) + i = 0 + while i < n: + if line[i] != "`": + i += 1 + continue + j = i + while j < n and line[j] == "`": + j += 1 + run = j - i + k = j + end = -1 + while k < n: + if line[k] != "`": + k += 1 + continue + m = k + while m < n and line[m] == "`": + m += 1 + if m - k == run: + end = m + break + k = m + if end < 0: + i = j + continue + for p in range(i, end): + out[p] = "\0" + i = end + return "".join(out) + + +class Converter: + def __init__(self, source: str, version: str, repo_url: str, pages: list[Page]): + self.source = source + self.version = version + self.repo_url = repo_url.rstrip("/") + self.by_path = {p.path: p for p in pages} + self.errors: list[str] = [] + + def tag_url(self, path: str) -> str: + base = f"{self.repo_url}/src/tag/v{self.version}" + return base if path in ("", ".") else f"{base}/{quote(path, safe='/')}" + + def resolve(self, src: str, target: str) -> str | None: + """The replacement for one relative link target, or None (and an error recorded).""" + path, hash_, anchor = target.partition("#") + suffix = hash_ + anchor + path = unquote(path) + base = "" if path.startswith("/") else posixpath.dirname(src) + joined = posixpath.normpath(posixpath.join(base, path.lstrip("/"))) + if joined == ".." or joined.startswith("../"): + self.errors.append(f"{src}: link '{target}' leaves the tree") + return None + page = self.by_path.get(joined) + if page is None and os.path.isdir(os.path.join(self.source, joined)): + index = "README.md" if joined == "." else f"{joined}/README.md" + page = self.by_path.get(index) + if page is not None: + return page.name + suffix + if os.path.exists(os.path.join(self.source, joined)): + return self.tag_url(joined) + suffix + self.errors.append(f"{src}: link '{target}' points at '{joined}', which does not exist in the tree") + return None + + def convert_line(self, src: str, line: str) -> str: + masked = mask_code_spans(line) + edits = [] + for match in _LINK.finditer(masked): + target = match.group(1) + if _SCHEME.match(target) or target.startswith(("#", "//")): + continue + replacement = self.resolve(src, line[match.start(1):match.end(1)]) + if replacement is not None: + edits.append((match.start(1), match.end(1), replacement)) + for start, end, replacement in reversed(edits): + line = line[:start] + replacement + line[end:] + return line + + def convert(self, src: str, text: str) -> str: + return "\n".join( + line if in_code else self.convert_line(src, line) for line, in_code in code_flags(text) + ) + + +def code_flags(text: str): + """Each line of the text with whether it belongs to a fenced code block (fences included).""" + fence = None + for line in text.split("\n"): + m = _FENCE.match(line) + if fence is None: + if m: + fence = m.group(1) + yield line, m is not None + continue + yield line, True + if m and m.group(1)[0] == fence[0] and len(m.group(1)) >= len(fence) and not line.strip().strip(fence[0]): + fence = None + + +def title_of(text: str, fallback: str) -> str: + for line, in_code in code_flags(text): + h = None if in_code else _H1.match(line) + if h: + return h.group(1).replace("[", "\\[").replace("]", "\\]") + return fallback + + +def sidebar(pages: list[Page], titles: dict[str, str]) -> str: + def link(page: Page) -> str: + return f"[{titles[page.path]}]({page.name})" + + lines = [] + home = next((p for p in pages if p.name == "Home" and p.segs is None), None) + install = next((p for p in pages if p.name == "Install" and p.segs is None), None) + for page in (home, install): + if page is not None: + lines.append(f"* {link(page)}") + + docs = [p for p in pages if p.segs is not None] + page_at = {p.segs: p for p in docs} + flat = [p for p in docs if len(p.segs) == 0 or (len(p.segs) == 1 and not p.index)] + tree = [p for p in docs if p not in flat] + + children: dict[tuple[str, ...], set[tuple[str, ...]]] = {} + for page in tree: + for i in range(1, len(page.segs) + 1): + children.setdefault(page.segs[: i - 1], set()).add(page.segs[:i]) + + def label(key: tuple[str, ...]) -> str: + page = page_at.get(key) + return link(page) if page else _capitalise(key[-1]) + + def render(key: tuple[str, ...], depth: int) -> None: + lines.append(" " * depth + f"* {label(key)}") + for child in sorted(children.get(key, ()), key=lambda k: k[-1].lower()): + render(child, depth + 1) + + if flat: + lines.append(f"* {FLAT_GROUP}") + for page in sorted(flat, key=lambda p: p.name.lower()): + lines.append(f" * {link(page)}") + + tops = children.get((), set()) + known = [(s,) for s in AREA_ORDER if (s,) in tops] + rest = sorted((k for k in tops if k[0] not in AREA_ORDER and k[0] != DEV_AREA), key=lambda k: k[0].lower()) + last = [(DEV_AREA,)] if (DEV_AREA,) in tops else [] + for key in known + rest + last: + render(key, 0) + return "\n".join(lines) + "\n" + + +def build(source: str, version: str, repo_url: str) -> tuple[dict[str, str], list[str]]: + """The wiki as {file name: content}, and every error found while building it.""" + pages, errors = collect_pages(source) + converter = Converter(source, version, repo_url, pages) + files: dict[str, str] = {} + titles: dict[str, str] = {} + for page in pages: + with open(os.path.join(source, page.path), encoding="utf-8", newline="") as fh: + text = fh.read() + titles[page.path] = title_of(text, page.name) + notice = ( + f"*Generated from [`{page.path}`]({converter.tag_url(page.path)}) at v{version}" + " – edit the repository, not this wiki.*" + ) + files[f"{page.name}.md"] = notice + "\n\n" + converter.convert(page.path, text) + files["_Sidebar.md"] = sidebar(pages, titles) + return files, errors + converter.errors + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--source", required=True, help="the tree to mirror") + parser.add_argument("--out", help="empty or missing directory that receives the wiki content") + parser.add_argument("--version", required=True, help="X.Y.Z of the tree; names the tag links point at") + parser.add_argument("--repo-url", default=DEFAULT_REPO_URL) + parser.add_argument("--check", action="store_true", help="convert, write nothing, fail on a problem") + args = parser.parse_args(argv) + + if not args.check and not args.out: + parser.error("--out is required unless --check is given") + if not os.path.isdir(args.source): + print(f"ERROR --source {args.source} is not a directory", file=sys.stderr) + return 1 + + files, errors = build(args.source, args.version, args.repo_url) + if errors: + for error in errors: + print(f"ERROR {error}", file=sys.stderr) + return 1 + + if not args.check: + if os.path.isdir(args.out) and os.listdir(args.out): + print(f"ERROR --out {args.out} is not empty", file=sys.stderr) + return 1 + os.makedirs(args.out, exist_ok=True) + for name, content in files.items(): + with open(os.path.join(args.out, name), "w", encoding="utf-8", newline="") as fh: + fh.write(content) + print(f"{len(files)} wiki files for v{args.version}" + (" (checked, nothing written)" if args.check else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index c1f384b..05f9566 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -194,6 +194,14 @@ jobs: tools/wikitool instructions verify tools/wikitool lint --fail-on-error + - name: Wiki mirror converts the docs + # The Gitea wiki mirrors docs/ at every release (release.yml, job wiki-mirror). A link the + # converter cannot resolve, or two pages that would share a wiki name, is cheaper to hear + # about here than when the release is cut. Writes nothing. + run: | + set -eu + python3 .gitea/scripts/wiki_mirror.py --check --source . --version "$(tr -d '[:space:]' < VERSION)" + - name: Version gate # A stack change with no version bump cannot be released, because the # release would carry a change nobody named. Scoped to what diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index ba1b27d..684bdc1 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -17,6 +17,13 @@ # instance issues - not a 1Password secret. Nothing here reaches outside the # instance, so the Zero-Trust secret path that the container-build workflows # use has nothing to carry. +# +# `wiki-mirror` is the second job: once `release` has published a release, it +# mirrors that tree's docs into the Gitea wiki (.gitea/scripts/wiki_mirror.py +# converts, wiki-push.sh pushes; wiki-mirror.yml does the same by hand for any +# tag). It is a separate job so that a failing mirror leaves the release and its +# tag standing: the release is the product, the wiki a view of it. It does not +# run for a `-beta.N` candidate, for which `release` sets `skip`. name: Release @@ -38,6 +45,10 @@ jobs: image: debian:trixie-slim permissions: contents: write + outputs: + skip: ${{ steps.version.outputs.skip }} + version: ${{ steps.version.outputs.version }} + tag: ${{ steps.version.outputs.tag }} env: WIKITOOL_SESSION_ID: release-${{ github.run_id }} WIKI_TRACE_DIR: /tmp/wikitool-trace @@ -233,3 +244,43 @@ jobs: done echo "Done: ${PUBLIC_BASE_URL}/${GITHUB_REPOSITORY}/releases/tag/${TAG}" + + wiki-mirror: + # Runs only after `release` succeeded and created a release (a candidate + # sets skip=true). `release` never depends on this job. + needs: release + if: needs.release.outputs.skip != 'true' + runs-on: linux-docker + container: + image: debian:trixie-slim + permissions: + contents: write + env: + VERSION: ${{ needs.release.outputs.version }} + TAG: ${{ needs.release.outputs.tag }} + PUBLIC_BASE_URL: https://gitea.nehmer.net + + steps: + - name: System dependencies + run: | + set -eu + apt-get update -qq + apt-get install -y --no-install-recommends python3 git nodejs ca-certificates curl jq + rm -rf /var/lib/apt/lists/* + + - uses: actions/checkout@v7 + + - name: Mirror the docs into the wiki + env: + WIKI_TOKEN: ${{ gitea.token }} + run: | + set -eu + python3 .gitea/scripts/wiki_mirror.py --source . --out /tmp/wiki-out \ + --version "$VERSION" --repo-url "${PUBLIC_BASE_URL}/${GITHUB_REPOSITORY}" + .gitea/scripts/wiki-push.sh /tmp/wiki-out "$VERSION" + + - name: Say which half failed + if: failure() + run: | + echo "Release ${TAG} is published; only the wiki mirror failed." + echo "The release and its tag stand. Dispatch wiki-mirror.yml with tag ${TAG} to retry the mirror." diff --git a/.gitea/workflows/wiki-mirror.yml b/.gitea/workflows/wiki-mirror.yml new file mode 100644 index 0000000..f697b7a --- /dev/null +++ b/.gitea/workflows/wiki-mirror.yml @@ -0,0 +1,78 @@ +# Mirror the docs of one release tag into the Gitea wiki, by hand. +# +# The same two scripts as the `wiki-mirror` job in release.yml, started on demand: +# to fill the wiki for the first time and look at it before the next release, +# and to catch up a mirror that failed after its release was already published. +# The wiki is a mirror and never a source - every run replaces its whole content, +# and nothing but CI writes to it. +# +# The converter and the push script come from the commit this workflow runs on +# (main); the docs come from the tag, checked out beside it. That way an old tag, +# whose tree predates the converter, can still be mirrored. +# +# Input `tag` defaults to the latest release, read from the release API. + +name: Wiki mirror + +on: + workflow_dispatch: + inputs: + tag: + description: 'Release tag to mirror (default: the latest release)' + required: false + default: '' + +jobs: + mirror: + runs-on: linux-docker + container: + image: debian:trixie-slim + permissions: + contents: write + env: + PUBLIC_BASE_URL: https://gitea.nehmer.net + + steps: + - name: System dependencies + run: | + set -eu + apt-get update -qq + apt-get install -y --no-install-recommends python3 git nodejs ca-certificates curl jq + rm -rf /var/lib/apt/lists/* + + - uses: actions/checkout@v7 + + - name: Resolve the tag + id: tag + env: + API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }} + TOKEN: ${{ gitea.token }} + INPUT_TAG: ${{ github.event.inputs.tag }} + run: | + set -eu + tag="$INPUT_TAG" + if [ -z "$tag" ]; then + tag="$(curl -sS -f -H "Authorization: token ${TOKEN}" "${API}/releases/latest" | jq -r '.tag_name')" + fi + case "$tag" in + v[0-9]*) ;; + *) echo "Tag '${tag}' is not a release tag (vX.Y.Z)."; exit 1 ;; + esac + echo "Mirroring ${tag}." + echo "tag=${tag}" >> "$GITHUB_OUTPUT" + echo "version=${tag#v}" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v7 + with: + ref: ${{ steps.tag.outputs.tag }} + path: tag-tree + + - name: Mirror the docs into the wiki + env: + VERSION: ${{ steps.tag.outputs.version }} + WIKI_TOKEN: ${{ gitea.token }} + run: | + set -eu + python3 .gitea/scripts/wiki_mirror.py --source tag-tree --out /tmp/wiki-out \ + --version "$VERSION" --repo-url "${PUBLIC_BASE_URL}/${GITHUB_REPOSITORY}" + .gitea/scripts/wiki-push.sh /tmp/wiki-out "$VERSION" diff --git a/CHANGES.md b/CHANGES.md index 8c6ef95..09d5ad8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,7 +14,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. --- -## 9.0.0-beta.12 - 2026-10-07 - tools/README.md: Asset-Modus beschreibt Entpacken in den eigenen Ordner, Wiederholung und Zurückrollen +## 9.0.0-beta.13 - 2026-10-07 - Gitea-Wiki als generierter Spiegel von docs/: Konverter, Push-Skript, Workflows **Author:** Torben Nehmer @@ -42,6 +42,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. - tools/README.md module tree names reference_doc.py and the fourth view cli_contract renders - Rebase-Review Gate: Anzeigename in Hilfe- und Fehlertexten von sync und publish angeglichen - tools/README.md: Asset-Modus beschreibt Entpacken in den eigenen Ordner, Wiederholung und Zurückrollen +- Gitea-Wiki als generierter Spiegel von docs/: Konverter, Push-Skript, Workflows ### Tutorials: installation in English, and a first source to a first answer (#191) diff --git a/README.md b/README.md index f9fe382..84e6a77 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,11 @@ Most days you need three of the how-to guides: [ingest a source](docs/how-to/ing [query the wiki](docs/how-to/query-the-wiki.md) and [maintain it](docs/how-to/maintain-the-wiki.md). How the parts fit together is [docs/explanation/architecture.md](docs/explanation/architecture.md). + +The Gitea wiki of this repository shows these pages for the latest release. It is generated from +`docs/` and not edited; how it is built is in [docs/dev/development.md](docs/dev/development.md). + + ## Your first ingestion Once the instance runs, [From your first source to your first answer](docs/tutorials/first-source-to-first-answer.md) diff --git a/VERSION b/VERSION index 7c4757e..f1d18b5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -9.0.0-beta.12 +9.0.0-beta.13 diff --git a/docs/dev/development.md b/docs/dev/development.md index 70dd8ca..3ce307d 100644 --- a/docs/dev/development.md +++ b/docs/dev/development.md @@ -20,6 +20,7 @@ them. The reason is a comment at `dist_cmd.DOCS_DEV_ONLY`, so that a later sessi - [A release feed in a non-public repository](#a-release-feed-in-a-non-public-repository) - [The release workflow](#the-release-workflow) - [The CI half](#the-ci-half) + - [The Gitea wiki](#the-gitea-wiki) - [How much of the stack the suite reaches](#how-much-of-the-stack-the-suite-reaches) - [Stack development as a session type of its own](#stack-development-as-a-session-type-of-its-own) @@ -155,6 +156,10 @@ the order a session actually goes through it: cause, publish, and start `release.yml` on `main` via `workflow_dispatch`. The check for an existing release prevents a second one. + A second job, `wiki-mirror`, runs after a suffix-free release and mirrors the docs of the + release tree into the Gitea wiki ([The Gitea wiki](#the-gitea-wiki) below). It is a job of its + own, so a failing mirror leaves the release and its tag standing and marks only itself red. + The three check commands are in step 3 above; what each of them checks is in [tools/CONTRACT.md](../../tools/CONTRACT.md), where `docs verify` holds it against the actual CLI. It is deliberately **not** repeated here: a second description of the same commands is @@ -170,7 +175,8 @@ runs the test suite, `docs verify`, `instructions verify` and a complete `setup- replay: a locally built release tarball, the preflight script with `--archive` in an empty directory, then the instruction's steps - the same path a new user actually takes, just without the download. `.gitea/workflows/nightly.yml` is the drift check against time rather than against -a commit. `.gitea/workflows/release.yml` is step 6 above. +a commit. `.gitea/workflows/release.yml` is step 6 above, and `.gitea/workflows/wiki-mirror.yml` +refills the Gitea wiki by hand (see below). Three more workflows carry the live tests of the tracker adapters (Super Productivity, CalDAV): `ci.yml` runs the CalDAV half in a step of its own against Radicale as a process, @@ -182,6 +188,35 @@ repository by hand. What the suite writes, how to run it against a tracker of yo a red run means is in [instructions/dev/tracker-testing.md](../../instructions/dev/tracker-testing.md). +### The Gitea wiki + +The wiki of `torben/chemenu` shows the documentation of the **latest release**. It is generated +and never edited: every run replaces its whole content, so a change made in the wiki is gone at +the next mirror, and nothing but CI writes to it (a push to the wiki repository by a session +would sit next to `publish`, which AGENTS.md invariant 5 forbids). + +`.gitea/scripts/wiki_mirror.py` converts a tree into the wiki's page set: every `*.md` under +`docs/` (the `dev/` pages included), plus `README.md` as `Home` and `INSTALL.md` as `Install`. +Wiki pages in sub-directories are not listed reliably, so the names are flat: the path below +`docs/` with `.md` dropped, each segment capitalised and joined by `-` +(`docs/how-to/ingest-a-source.md` becomes `How-to-Ingest-a-source`; a `README.md` stands for its +directory). Two sources that would get the same name, compared without case, abort the run. +Relative links between mirrored files become wiki page names, relative links to anything else +become links to that file at the release tag, and a link whose target does not exist aborts the +run. A `_Sidebar` is generated, and every page opens with a line saying which file at which +version it came from. `.gitea/scripts/wiki-push.sh` then makes the wiki repository hold exactly +that output - a page the converter no longer produces is removed - and does not commit when +nothing changed. + +Two things start it: the `wiki-mirror` job of `release.yml`, after each release (never for a +candidate), and `.gitea/workflows/wiki-mirror.yml` by hand with `workflow_dispatch`. The manual +workflow takes a `tag` (default: the latest release) and runs the converter and push script of +`main` against that tag's tree. Use it to fill the wiki before the first release that carries the +job, or to catch up a mirror that failed after its release was published. `ci.yml` runs +`wiki_mirror.py --check` against the working tree on every push, so a link the mirror could not +resolve fails there rather than at the release. The scripts live under `.gitea/`, which +`dist export` does not copy, so none of this reaches an instance. + ### How much of the stack the suite reaches Coverage is measured in CI - `pytest --cov`, config in `tools/.coveragerc`, HTML and XML diff --git a/tools/chemenu/tests/test_wiki_mirror.py b/tools/chemenu/tests/test_wiki_mirror.py new file mode 100644 index 0000000..94600aa --- /dev/null +++ b/tools/chemenu/tests/test_wiki_mirror.py @@ -0,0 +1,313 @@ +"""Tests for the Gitea wiki mirror: the converter `.gitea/scripts/wiki_mirror.py` and the push +script `.gitea/scripts/wiki-push.sh`. Neither is part of `tools/chemenu`; the converter is loaded +by path and the push script is run against a bare repository on disk.""" +from __future__ import annotations + +import importlib.util +import os +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[3] +SCRIPTS = REPO / ".gitea" / "scripts" +TAG_URL = "https://gitea.nehmer.net/torben/chemenu/src/tag/v1.2.3" + + +def _load(): + spec = importlib.util.spec_from_file_location("wiki_mirror", SCRIPTS / "wiki_mirror.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +wm = _load() + + +def _write(root: Path, files: dict[str, str]) -> Path: + for rel, text in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return root + + +def _run(source: Path, out: Path | None, *extra: str) -> int: + argv = ["--source", str(source), "--version", "1.2.3", *extra] + if out is not None: + argv += ["--out", str(out)] + return wm.main(argv) + + +def _read_all(out: Path) -> dict[str, bytes]: + return {p.name: p.read_bytes() for p in sorted(out.iterdir())} + + +MINI = { + "README.md": "# Home\n\n[guide](docs/how-to/ingest.md) [agents](AGENTS.md)\n", + "INSTALL.md": "# Install\n", + "AGENTS.md": "# Agents\n", + "tools/CONTRACT.md": "# Contract\n", + "docs/how-to/README.md": "# How-to\n", + "docs/how-to/ingest.md": "# Ingest\n\n[up](README.md) [ref](../reference/commands/pages.md#foo)\n", + "docs/reference/commands/README.md": "# Commands\n", + "docs/reference/commands/pages.md": "# Pages\n\n## Foo\n", + "docs/dev/development.md": "# Developing\n", +} + + +@pytest.mark.parametrize( + "path, name", + [ + ("docs/how-to/ingest-a-source.md", "How-to-Ingest-a-source"), + ("docs/how-to/README.md", "How-to"), + ("docs/reference/commands/pages.md", "Reference-Commands-Pages"), + ("docs/reference/commands/README.md", "Reference-Commands"), + ("docs/dev/development.md", "Dev-Development"), + ("docs/why-gates-are-code.md", "Why-gates-are-code"), + ("README.md", "Home"), + ("INSTALL.md", "Install"), + ], +) +def test_name_rule(tmp_path, path, name): + _write(tmp_path, {"README.md": "x\n", "INSTALL.md": "x\n", path: "x\n"}) + pages, errors = wm.collect_pages(str(tmp_path)) + assert errors == [] + assert {p.path: p.name for p in pages}[path] == name + + +def test_every_file_under_docs_gets_a_unique_name_and_output_is_reproducible(tmp_path): + pages, errors = wm.collect_pages(str(REPO)) + assert errors == [] + mirrored = {p.path for p in pages} + on_disk = {p.relative_to(REPO).as_posix() for p in (REPO / "docs").rglob("*.md")} + assert on_disk <= mirrored + names = [p.name.lower() for p in pages] + assert len(names) == len(set(names)) + + first, second = tmp_path / "a", tmp_path / "b" + assert _run(REPO, first) == 0 + assert _run(REPO, second) == 0 + assert _read_all(first) == _read_all(second) + + +def test_every_relative_link_in_the_output_is_a_page_or_a_tag_url(tmp_path): + out = tmp_path / "out" + assert _run(REPO, out) == 0 + names = {p.stem for p in out.iterdir()} + for page in out.iterdir(): + for line, in_code in wm.code_flags(page.read_text(encoding="utf-8")): + if in_code: + continue + for match in wm._LINK.finditer(wm.mask_code_spans(line)): + target = match.group(1) + if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", target) or target.startswith("#"): + continue + assert target.partition("#")[0] in names, (page.name, target) + sidebar = (out / "_Sidebar.md").read_text(encoding="utf-8") + linked = set(re.findall(r"\]\(([^)#]+)\)", sidebar)) + assert linked <= names + + +def test_links_are_rewritten_and_everything_else_is_left_alone(tmp_path): + out = tmp_path / "out" + assert _run(_write(tmp_path / "src", MINI), out) == 0 + ingest = (out / "How-to-Ingest.md").read_text(encoding="utf-8") + assert "[up](How-to)" in ingest + assert "[ref](Reference-Commands-Pages#foo)" in ingest + home = (out / "Home.md").read_text(encoding="utf-8") + assert "[guide](How-to-Ingest)" in home + assert f"[agents]({TAG_URL}/AGENTS.md)" in home + assert home.startswith( + f"*Generated from [`README.md`]({TAG_URL}/README.md) at v1.2.3 – edit the repository, not this wiki.*\n\n" + ) + assert sorted(_read_all(out)) == sorted( + [ + "Home.md", "Install.md", "How-to.md", "How-to-Ingest.md", "Reference-Commands.md", + "Reference-Commands-Pages.md", "Dev-Development.md", "_Sidebar.md", + ] + ) + + +def test_external_links_anchors_and_code_are_not_touched(tmp_path): + body = ( + "[ext](https://example.org/a.md) [mail](mailto:a@b.c) [here](#foo)\n" + "`[inline](missing.md)` and [`code text`](AGENTS.md)\n" + "```\n[fenced](missing.md)\n```\n" + "~~~sh\n[tilde](missing.md)\n~~~\n" + " ````\n```\n[nested](missing.md)\n````\n" + ) + src = _write(tmp_path / "src", {"README.md": body, "AGENTS.md": "x\n"}) + out = tmp_path / "out" + assert _run(src, out) == 0 + home = (out / "Home.md").read_text(encoding="utf-8").split("\n\n", 1)[1] + assert home == body.replace("[`code text`](AGENTS.md)", f"[`code text`]({TAG_URL}/AGENTS.md)") + + +def test_link_to_a_directory_without_a_readme_points_at_the_tag(tmp_path): + src = _write(tmp_path / "src", {"README.md": "[d](docs/) [t](tools/)\n", "tools/x.txt": "x\n", "docs/a.md": "x\n"}) + out = tmp_path / "out" + assert _run(src, out) == 0 + home = (out / "Home.md").read_text(encoding="utf-8") + assert f"[d]({TAG_URL}/docs)" in home and f"[t]({TAG_URL}/tools)" in home + + +@pytest.mark.parametrize( + "files, expect", + [ + ({"docs/a/b.md": "x\n", "docs/a-b.md": "x\n"}, ["docs/a/b.md", "docs/a-b.md"]), + ({"docs/how-to/x.md": "x\n", "docs/How-to/x.md": "x\n"}, ["docs/how-to/x.md", "docs/How-to/x.md"]), + ({"docs/install.md": "x\n"}, ["docs/install.md", "reserved"]), + ({"docs/_sidebar.md": "x\n"}, ["docs/_sidebar.md", "reserved"]), + ], +) +def test_a_name_collision_fails_and_names_the_paths(tmp_path, capsys, files, expect): + src = _write(tmp_path / "src", {"README.md": "x\n", "INSTALL.md": "x\n", **files}) + assert _run(src, tmp_path / "out") == 1 + err = capsys.readouterr().err + for fragment in expect: + assert fragment in err + assert not (tmp_path / "out").exists() + + +def test_a_link_to_a_missing_file_fails_and_names_both_paths(tmp_path, capsys): + src = _write(tmp_path / "src", {"README.md": "x\n", "docs/a.md": "[gone](../nowhere/x.md)\n"}) + assert _run(src, tmp_path / "out") == 1 + err = capsys.readouterr().err + assert "docs/a.md" in err and "nowhere/x.md" in err + + +def test_a_link_out_of_the_tree_fails(tmp_path, capsys): + src = _write(tmp_path / "src", {"README.md": "[x](../../etc/passwd)\n"}) + assert _run(src, tmp_path / "out") == 1 + assert "leaves the tree" in capsys.readouterr().err + + +def test_check_writes_nothing_and_fails_on_a_problem(tmp_path): + src = _write(tmp_path / "src", MINI) + assert _run(src, None, "--check") == 0 + assert not (tmp_path / "out").exists() + _write(src, {"docs/bad.md": "[x](missing.md)\n"}) + assert _run(src, None, "--check") == 1 + + +def test_a_flat_docs_directory_without_areas_is_mirrored(tmp_path): + src = _write( + tmp_path / "src", + { + "README.md": "[why](docs/why.md) [dev](DEVELOPMENT.md)\n", + "DEVELOPMENT.md": "x\n", + "docs/why.md": "# Why\n\n[home](../README.md)\n", + }, + ) + out = tmp_path / "out" + assert _run(src, out) == 0 + assert "[why](Why)" in (out / "Home.md").read_text(encoding="utf-8") + assert f"[dev]({TAG_URL}/DEVELOPMENT.md)" in (out / "Home.md").read_text(encoding="utf-8") + assert "[home](Home)" in (out / "Why.md").read_text(encoding="utf-8") + sidebar = (out / "_Sidebar.md").read_text(encoding="utf-8") + assert "* Documentation\n * [Why](Why)\n" in sidebar + + +def test_sidebar_order_and_nesting(tmp_path): + out = tmp_path / "out" + assert _run(_write(tmp_path / "src", MINI), out) == 0 + assert (out / "_Sidebar.md").read_text(encoding="utf-8") == ( + "* [Home](Home)\n" + "* [Install](Install)\n" + "* [How-to](How-to)\n" + " * [Ingest](How-to-Ingest)\n" + "* Reference\n" + " * [Commands](Reference-Commands)\n" + " * [Pages](Reference-Commands-Pages)\n" + "* Dev\n" + " * [Developing](Dev-Development)\n" + ) + + +def test_a_non_empty_output_directory_is_refused(tmp_path, capsys): + out = tmp_path / "out" + _write(out, {"stale.md": "x\n"}) + assert _run(_write(tmp_path / "src", MINI), out) == 1 + assert "not empty" in capsys.readouterr().err + + +# --- wiki-push.sh ----------------------------------------------------------------------------- + +needs_tools = pytest.mark.skipif( + not all(shutil.which(t) for t in ("git", "sh", "base64", "diff", "find")), + reason="needs git and POSIX utilities", +) + + +def _git(*args: str, cwd: Path | None = None) -> str: + return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout + + +class Wiki: + def __init__(self, root: Path): + self.bare = root / "wiki.git" + self.out = root / "out" + subprocess.run(["git", "init", "-q", "--bare", str(self.bare)], check=True) + + def mirror(self, files: dict[str, str], version: str = "1.0.0", token: str = "t"): + shutil.rmtree(self.out, ignore_errors=True) + _write(self.out, files) + env = {**os.environ, "WIKI_REMOTE": str(self.bare), "WIKI_TOKEN": token} + if not token: + env.pop("WIKI_TOKEN") + return subprocess.run( + ["sh", str(SCRIPTS / "wiki-push.sh"), str(self.out), version], + env=env, capture_output=True, text=True, + ) + + def commits(self) -> int: + refs = _git("--git-dir", str(self.bare), "for-each-ref") + return int(_git("--git-dir", str(self.bare), "rev-list", "--all", "--count")) if refs else 0 + + def files(self) -> dict[str, str]: + branch = _git("--git-dir", str(self.bare), "for-each-ref", "--format=%(refname)").split()[0] + names = _git("--git-dir", str(self.bare), "ls-tree", "-r", "--name-only", branch).split() + return {n: _git("--git-dir", str(self.bare), "show", f"{branch}:{n}") for n in names} + + +@needs_tools +def test_push_fills_an_empty_wiki_with_exactly_the_output(tmp_path): + wiki = Wiki(tmp_path) + result = wiki.mirror({"Home.md": "home\n", "A.md": "a\n", "_Sidebar.md": "s\n"}) + assert result.returncode == 0, result.stderr + assert wiki.files() == {"Home.md": "home\n", "A.md": "a\n", "_Sidebar.md": "s\n"} + assert wiki.commits() == 1 + + +@needs_tools +def test_push_of_the_same_output_makes_no_commit(tmp_path): + wiki = Wiki(tmp_path) + files = {"Home.md": "home\n", "A.md": "a\n"} + assert wiki.mirror(files).returncode == 0 + again = wiki.mirror(files) + assert again.returncode == 0 + assert "Wiki already matches v1.0.0" in again.stdout + assert wiki.commits() == 1 + + +@needs_tools +def test_push_removes_a_page_the_output_no_longer_has_and_updates_the_rest(tmp_path): + wiki = Wiki(tmp_path) + assert wiki.mirror({"Home.md": "one\n", "Gone.md": "x\n", "Keep.md": "k\n"}).returncode == 0 + assert wiki.mirror({"Home.md": "two\n", "Keep.md": "k\n"}, version="2.0.0").returncode == 0 + assert wiki.files() == {"Home.md": "two\n", "Keep.md": "k\n"} + assert wiki.commits() == 2 + assert "Mirror docs at v2.0.0" in _git("--git-dir", str(wiki.bare), "log", "-1", "--format=%s", "--all") + + +@needs_tools +def test_push_without_a_token_fails_before_touching_the_wiki(tmp_path): + wiki = Wiki(tmp_path) + result = wiki.mirror({"Home.md": "home\n"}, token="") + assert result.returncode != 0 + assert "WIKI_TOKEN" in result.stderr + assert wiki.commits() == 0