feat(workflow): Gitea-Wiki als generierter Spiegel von docs/ (#192)

Konverter .gitea/scripts/wiki_mirror.py, Push-Skript wiki-push.sh, Job wiki-mirror in release.yml, Dispatch-Workflow wiki-mirror.yml und --check-Schritt in ci.yml. Version 9.0.0-beta.13.

Files changed:
- .gitea/scripts/wiki-push.sh
- .gitea/scripts/wiki_mirror.py
- .gitea/workflows/ci.yml
- .gitea/workflows/release.yml
- .gitea/workflows/wiki-mirror.yml
- CHANGES.md
- README.md
- VERSION
- docs/dev/development.md
- tools/chemenu/tests/test_wiki_mirror.py

Co-Authored-By: Claude Code <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2

.gitea/scripts/wiki-push.sh

@@ -0,0 +1,82 @@
+#!/bin/sh
+# Make the wiki repository of this repo hold exactly the converter output.
+#
+#   wiki-push.sh <output-dir> <version>
+#
+# Environment:
+#   WIKI_TOKEN          token with write access to the wiki (required)
+#   GITHUB_SERVER_URL   set by the runner; GITHUB_REPOSITORY likewise
+#   WIKI_REMOTE         overrides the wiki clone URL (the tests point it at a local repository)
+#
+# The wiki is a mirror, never a source: every file in it is replaced by the converter output, so
+# a page the converter no longer produces disappears. Afterwards the working tree minus .git/ and
+# the output directory are compared, and any difference fails the script before anything is
+# pushed. An unchanged result ends with exit 0 and no commit.
+set -eu
+
+out="${1:?usage: wiki-push.sh <output-dir> <version>}"
+version="${2:?usage: wiki-push.sh <output-dir> <version>}"
+
+if [ -z "${WIKI_TOKEN:-}" ]; then
+  echo "ERROR WIKI_TOKEN is empty: the wiki mirror needs a token that may push to the wiki repository." >&2
+  echo "      The job token (gitea.token) is tried first; if Gitea refuses it, the operator adds a" >&2
+  echo "      repository secret with write access and the workflow passes that one instead." >&2
+  exit 1
+fi
+[ -f "$out/Home.md" ] || { echo "ERROR $out/Home.md is missing - not a converter output" >&2; exit 1; }
+
+server="${GITHUB_SERVER_URL:-}"
+repo="${GITHUB_REPOSITORY:-}"
+remote="${WIKI_REMOTE:-${server}/${repo}.wiki.git}"
+api="${server}/api/v1/repos/${repo}"
+
+# The token travels in a header, not in the URL, so it appears in no git message or process list.
+auth="Authorization: Basic $(printf 'wiki-mirror:%s' "$WIKI_TOKEN" | base64 | tr -d '\n')"
+git_remote() { git -c "http.extraheader=${auth}" "$@"; }
+
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+wiki="$work/wiki"
+
+if ! git_remote clone -q "$remote" "$wiki" 2>"$work/clone.err"; then
+  echo "Cloning ${remote} failed:"
+  cat "$work/clone.err"
+  # A wiki that has never had a page has no repository yet; the first page created through
+  # the API brings it into existence.
+  echo "Creating the first page through the API to bring the wiki repository into existence."
+  content="$(base64 < "$out/Home.md" | tr -d '\n')"
+  jq -n --arg title Home --arg content "$content" --arg message "Mirror docs at v${version}" \
+    '{title: $title, content_base64: $content, message: $message}' > "$work/first-page.json"
+  status="$(curl -sS -o "$work/first-page.out" -w '%{http_code}' -X POST "${api}/wiki/new" \
+    -H "Authorization: token ${WIKI_TOKEN}" -H 'Content-Type: application/json' \
+    --data-binary @"$work/first-page.json")"
+  case "$status" in
+    2??) ;;
+    *)
+      echo "ERROR the API refused the first page (HTTP ${status}):" >&2
+      cat "$work/first-page.out" >&2
+      exit 1
+      ;;
+  esac
+  git_remote clone -q "$remote" "$wiki"
+fi
+
+find "$wiki" -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} +
+cp -R "$out"/. "$wiki"/
+git -C "$wiki" add -A
+
+if ! diff -r -x .git "$out" "$wiki" > "$work/diff.txt"; then
+  echo "ERROR the wiki working tree differs from the converter output after copying:" >&2
+  cat "$work/diff.txt" >&2
+  exit 1
+fi
+
+if git -C "$wiki" diff --cached --quiet; then
+  echo "Wiki already matches v${version}"
+  exit 0
+fi
+
+git -C "$wiki" -c user.name='Wiki Mirror' -c user.email='wiki-mirror@noreply.invalid' \
+  commit -q -m "Mirror docs at v${version}"
+git_remote -C "$wiki" push -q origin HEAD
+echo "Wiki now mirrors v${version}"

.gitea/scripts/wiki_mirror.py

@@ -0,0 +1,293 @@
+#!/usr/bin/env python3
+"""Convert the documentation of one release tree into the flat page set of a Gitea wiki.
+
+    wiki_mirror.py --source <tree> --out <dir> --version <X.Y.Z> [--repo-url <url>] [--check]
+
+The output directory becomes the whole content of the wiki repository, so the conversion is
+deterministic: the same tree and version give the same bytes. `--check` converts in memory,
+writes nothing and fails only on a name collision or an unresolvable relative link.
+
+Plain standard library on purpose: it runs in CI before any tool environment exists, and it
+must work against the tree of any past tag.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import posixpath
+import re
+import sys
+from urllib.parse import quote, unquote
+
+DEFAULT_REPO_URL = "https://gitea.nehmer.net/torben/chemenu"
+RESERVED_NAMES = {"home", "install", "_sidebar"}
+AREA_ORDER = ("tutorials", "how-to", "reference", "explanation")
+DEV_AREA = "dev"
+FLAT_GROUP = "Documentation"
+
+_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:")
+_FENCE = re.compile(r"^\s*(`{3,}|~{3,})")
+_LINK = re.compile(r"""\]\(\s*([^)\s<][^)\s]*)((?:\s+(?:"[^"]*"|'[^']*'))?)\s*\)""")
+_H1 = re.compile(r"^# +(.+?)\s*#*\s*$")
+
+
+class Page:
+    def __init__(self, path: str, name: str, segs: tuple[str, ...] | None, index: bool):
+        self.path = path
+        self.name = name
+        self.segs = segs  # sidebar position below docs/; None for Home and Install
+        self.index = index  # a README.md: stands for its directory
+
+
+def _capitalise(segment: str) -> str:
+    return segment[:1].upper() + segment[1:]
+
+
+def collect_pages(source: str) -> tuple[list[Page], list[str]]:
+    """Every page to mirror, in a stable order, plus the collisions found among their names."""
+    pages: list[Page] = []
+    if os.path.isfile(os.path.join(source, "README.md")):
+        pages.append(Page("README.md", "Home", None, False))
+    if os.path.isfile(os.path.join(source, "INSTALL.md")):
+        pages.append(Page("INSTALL.md", "Install", None, False))
+
+    docs = os.path.join(source, "docs")
+    found: list[str] = []
+    for base, dirs, files in os.walk(docs):
+        dirs.sort()
+        for file in sorted(files):
+            if file.endswith(".md"):
+                rel = os.path.relpath(os.path.join(base, file), source)
+                found.append(rel.replace(os.sep, "/"))
+    for path in sorted(found):
+        parts = path[len("docs/"):-len(".md")].split("/")
+        index = parts[-1] == "README"
+        if index:
+            parts = parts[:-1]
+        name = "-".join(_capitalise(p) for p in parts) if parts else FLAT_GROUP
+        pages.append(Page(path, name, tuple(parts), index))
+
+    errors: list[str] = []
+    seen: dict[str, str] = {}
+    for page in pages:
+        key = page.name.lower()
+        if key in RESERVED_NAMES and page.segs is not None:
+            errors.append(f"{page.path}: page name '{page.name}' is reserved for a generated page")
+        elif key in seen:
+            errors.append(
+                f"name collision on '{page.name}' (compared without case): {seen[key]} and {page.path}"
+            )
+        else:
+            seen[key] = page.path
+    return pages, errors
+
+
+def mask_code_spans(line: str) -> str:
+    """The line with every inline code span blanked to NULs, same length, so offsets still match."""
+    out = list(line)
+    n = len(line)
+    i = 0
+    while i < n:
+        if line[i] != "`":
+            i += 1
+            continue
+        j = i
+        while j < n and line[j] == "`":
+            j += 1
+        run = j - i
+        k = j
+        end = -1
+        while k < n:
+            if line[k] != "`":
+                k += 1
+                continue
+            m = k
+            while m < n and line[m] == "`":
+                m += 1
+            if m - k == run:
+                end = m
+                break
+            k = m
+        if end < 0:
+            i = j
+            continue
+        for p in range(i, end):
+            out[p] = "\0"
+        i = end
+    return "".join(out)
+
+
+class Converter:
+    def __init__(self, source: str, version: str, repo_url: str, pages: list[Page]):
+        self.source = source
+        self.version = version
+        self.repo_url = repo_url.rstrip("/")
+        self.by_path = {p.path: p for p in pages}
+        self.errors: list[str] = []
+
+    def tag_url(self, path: str) -> str:
+        base = f"{self.repo_url}/src/tag/v{self.version}"
+        return base if path in ("", ".") else f"{base}/{quote(path, safe='/')}"
+
+    def resolve(self, src: str, target: str) -> str | None:
+        """The replacement for one relative link target, or None (and an error recorded)."""
+        path, hash_, anchor = target.partition("#")
+        suffix = hash_ + anchor
+        path = unquote(path)
+        base = "" if path.startswith("/") else posixpath.dirname(src)
+        joined = posixpath.normpath(posixpath.join(base, path.lstrip("/")))
+        if joined == ".." or joined.startswith("../"):
+            self.errors.append(f"{src}: link '{target}' leaves the tree")
+            return None
+        page = self.by_path.get(joined)
+        if page is None and os.path.isdir(os.path.join(self.source, joined)):
+            index = "README.md" if joined == "." else f"{joined}/README.md"
+            page = self.by_path.get(index)
+        if page is not None:
+            return page.name + suffix
+        if os.path.exists(os.path.join(self.source, joined)):
+            return self.tag_url(joined) + suffix
+        self.errors.append(f"{src}: link '{target}' points at '{joined}', which does not exist in the tree")
+        return None
+
+    def convert_line(self, src: str, line: str) -> str:
+        masked = mask_code_spans(line)
+        edits = []
+        for match in _LINK.finditer(masked):
+            target = match.group(1)
+            if _SCHEME.match(target) or target.startswith(("#", "//")):
+                continue
+            replacement = self.resolve(src, line[match.start(1):match.end(1)])
+            if replacement is not None:
+                edits.append((match.start(1), match.end(1), replacement))
+        for start, end, replacement in reversed(edits):
+            line = line[:start] + replacement + line[end:]
+        return line
+
+    def convert(self, src: str, text: str) -> str:
+        return "\n".join(
+            line if in_code else self.convert_line(src, line) for line, in_code in code_flags(text)
+        )
+
+
+def code_flags(text: str):
+    """Each line of the text with whether it belongs to a fenced code block (fences included)."""
+    fence = None
+    for line in text.split("\n"):
+        m = _FENCE.match(line)
+        if fence is None:
+            if m:
+                fence = m.group(1)
+            yield line, m is not None
+            continue
+        yield line, True
+        if m and m.group(1)[0] == fence[0] and len(m.group(1)) >= len(fence) and not line.strip().strip(fence[0]):
+            fence = None
+
+
+def title_of(text: str, fallback: str) -> str:
+    for line, in_code in code_flags(text):
+        h = None if in_code else _H1.match(line)
+        if h:
+            return h.group(1).replace("[", "\\[").replace("]", "\\]")
+    return fallback
+
+
+def sidebar(pages: list[Page], titles: dict[str, str]) -> str:
+    def link(page: Page) -> str:
+        return f"[{titles[page.path]}]({page.name})"
+
+    lines = []
+    home = next((p for p in pages if p.name == "Home" and p.segs is None), None)
+    install = next((p for p in pages if p.name == "Install" and p.segs is None), None)
+    for page in (home, install):
+        if page is not None:
+            lines.append(f"* {link(page)}")
+
+    docs = [p for p in pages if p.segs is not None]
+    page_at = {p.segs: p for p in docs}
+    flat = [p for p in docs if len(p.segs) == 0 or (len(p.segs) == 1 and not p.index)]
+    tree = [p for p in docs if p not in flat]
+
+    children: dict[tuple[str, ...], set[tuple[str, ...]]] = {}
+    for page in tree:
+        for i in range(1, len(page.segs) + 1):
+            children.setdefault(page.segs[: i - 1], set()).add(page.segs[:i])
+
+    def label(key: tuple[str, ...]) -> str:
+        page = page_at.get(key)
+        return link(page) if page else _capitalise(key[-1])
+
+    def render(key: tuple[str, ...], depth: int) -> None:
+        lines.append("  " * depth + f"* {label(key)}")
+        for child in sorted(children.get(key, ()), key=lambda k: k[-1].lower()):
+            render(child, depth + 1)
+
+    if flat:
+        lines.append(f"* {FLAT_GROUP}")
+        for page in sorted(flat, key=lambda p: p.name.lower()):
+            lines.append(f"  * {link(page)}")
+
+    tops = children.get((), set())
+    known = [(s,) for s in AREA_ORDER if (s,) in tops]
+    rest = sorted((k for k in tops if k[0] not in AREA_ORDER and k[0] != DEV_AREA), key=lambda k: k[0].lower())
+    last = [(DEV_AREA,)] if (DEV_AREA,) in tops else []
+    for key in known + rest + last:
+        render(key, 0)
+    return "\n".join(lines) + "\n"
+
+
+def build(source: str, version: str, repo_url: str) -> tuple[dict[str, str], list[str]]:
+    """The wiki as {file name: content}, and every error found while building it."""
+    pages, errors = collect_pages(source)
+    converter = Converter(source, version, repo_url, pages)
+    files: dict[str, str] = {}
+    titles: dict[str, str] = {}
+    for page in pages:
+        with open(os.path.join(source, page.path), encoding="utf-8", newline="") as fh:
+            text = fh.read()
+        titles[page.path] = title_of(text, page.name)
+        notice = (
+            f"*Generated from [`{page.path}`]({converter.tag_url(page.path)}) at v{version}"
+            " – edit the repository, not this wiki.*"
+        )
+        files[f"{page.name}.md"] = notice + "\n\n" + converter.convert(page.path, text)
+    files["_Sidebar.md"] = sidebar(pages, titles)
+    return files, errors + converter.errors
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    parser.add_argument("--source", required=True, help="the tree to mirror")
+    parser.add_argument("--out", help="empty or missing directory that receives the wiki content")
+    parser.add_argument("--version", required=True, help="X.Y.Z of the tree; names the tag links point at")
+    parser.add_argument("--repo-url", default=DEFAULT_REPO_URL)
+    parser.add_argument("--check", action="store_true", help="convert, write nothing, fail on a problem")
+    args = parser.parse_args(argv)
+
+    if not args.check and not args.out:
+        parser.error("--out is required unless --check is given")
+    if not os.path.isdir(args.source):
+        print(f"ERROR --source {args.source} is not a directory", file=sys.stderr)
+        return 1
+
+    files, errors = build(args.source, args.version, args.repo_url)
+    if errors:
+        for error in errors:
+            print(f"ERROR {error}", file=sys.stderr)
+        return 1
+
+    if not args.check:
+        if os.path.isdir(args.out) and os.listdir(args.out):
+            print(f"ERROR --out {args.out} is not empty", file=sys.stderr)
+            return 1
+        os.makedirs(args.out, exist_ok=True)
+        for name, content in files.items():
+            with open(os.path.join(args.out, name), "w", encoding="utf-8", newline="") as fh:
+                fh.write(content)
+    print(f"{len(files)} wiki files for v{args.version}" + (" (checked, nothing written)" if args.check else ""))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.gitea/workflows/ci.yml

@@ -194,6 +194,14 @@ jobs:
           tools/wikitool instructions verify
           tools/wikitool lint --fail-on-error
 
+      - name: Wiki mirror converts the docs
+        # The Gitea wiki mirrors docs/ at every release (release.yml, job wiki-mirror). A link the
+        # converter cannot resolve, or two pages that would share a wiki name, is cheaper to hear
+        # about here than when the release is cut. Writes nothing.
+        run: |
+          set -eu
+          python3 .gitea/scripts/wiki_mirror.py --check --source . --version "$(tr -d '[:space:]' < VERSION)"
+
       - name: Version gate
         # A stack change with no version bump cannot be released, because the
         # release would carry a change nobody named. Scoped to what

.gitea/workflows/release.yml

@@ -17,6 +17,13 @@
 # instance issues - not a 1Password secret. Nothing here reaches outside the
 # instance, so the Zero-Trust secret path that the container-build workflows
 # use has nothing to carry.
+#
+# `wiki-mirror` is the second job: once `release` has published a release, it
+# mirrors that tree's docs into the Gitea wiki (.gitea/scripts/wiki_mirror.py
+# converts, wiki-push.sh pushes; wiki-mirror.yml does the same by hand for any
+# tag). It is a separate job so that a failing mirror leaves the release and its
+# tag standing: the release is the product, the wiki a view of it. It does not
+# run for a `-beta.N` candidate, for which `release` sets `skip`.
 
 name: Release
 
@@ -38,6 +45,10 @@ jobs:
       image: debian:trixie-slim
     permissions:
       contents: write
+    outputs:
+      skip: ${{ steps.version.outputs.skip }}
+      version: ${{ steps.version.outputs.version }}
+      tag: ${{ steps.version.outputs.tag }}
     env:
       WIKITOOL_SESSION_ID: release-${{ github.run_id }}
       WIKI_TRACE_DIR: /tmp/wikitool-trace
@@ -233,3 +244,43 @@ jobs:
           done
 
           echo "Done: ${PUBLIC_BASE_URL}/${GITHUB_REPOSITORY}/releases/tag/${TAG}"
+
+  wiki-mirror:
+    # Runs only after `release` succeeded and created a release (a candidate
+    # sets skip=true). `release` never depends on this job.
+    needs: release
+    if: needs.release.outputs.skip != 'true'
+    runs-on: linux-docker
+    container:
+      image: debian:trixie-slim
+    permissions:
+      contents: write
+    env:
+      VERSION: ${{ needs.release.outputs.version }}
+      TAG: ${{ needs.release.outputs.tag }}
+      PUBLIC_BASE_URL: https://gitea.nehmer.net
+
+    steps:
+      - name: System dependencies
+        run: |
+          set -eu
+          apt-get update -qq
+          apt-get install -y --no-install-recommends python3 git nodejs ca-certificates curl jq
+          rm -rf /var/lib/apt/lists/*
+
+      - uses: actions/checkout@v7
+
+      - name: Mirror the docs into the wiki
+        env:
+          WIKI_TOKEN: ${{ gitea.token }}
+        run: |
+          set -eu
+          python3 .gitea/scripts/wiki_mirror.py --source . --out /tmp/wiki-out \
+            --version "$VERSION" --repo-url "${PUBLIC_BASE_URL}/${GITHUB_REPOSITORY}"
+          .gitea/scripts/wiki-push.sh /tmp/wiki-out "$VERSION"
+
+      - name: Say which half failed
+        if: failure()
+        run: |
+          echo "Release ${TAG} is published; only the wiki mirror failed."
+          echo "The release and its tag stand. Dispatch wiki-mirror.yml with tag ${TAG} to retry the mirror."

.gitea/workflows/wiki-mirror.yml

@@ -0,0 +1,78 @@
+# Mirror the docs of one release tag into the Gitea wiki, by hand.
+#
+# The same two scripts as the `wiki-mirror` job in release.yml, started on demand:
+# to fill the wiki for the first time and look at it before the next release,
+# and to catch up a mirror that failed after its release was already published.
+# The wiki is a mirror and never a source - every run replaces its whole content,
+# and nothing but CI writes to it.
+#
+# The converter and the push script come from the commit this workflow runs on
+# (main); the docs come from the tag, checked out beside it. That way an old tag,
+# whose tree predates the converter, can still be mirrored.
+#
+# Input `tag` defaults to the latest release, read from the release API.
+
+name: Wiki mirror
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Release tag to mirror (default: the latest release)'
+        required: false
+        default: ''
+
+jobs:
+  mirror:
+    runs-on: linux-docker
+    container:
+      image: debian:trixie-slim
+    permissions:
+      contents: write
+    env:
+      PUBLIC_BASE_URL: https://gitea.nehmer.net
+
+    steps:
+      - name: System dependencies
+        run: |
+          set -eu
+          apt-get update -qq
+          apt-get install -y --no-install-recommends python3 git nodejs ca-certificates curl jq
+          rm -rf /var/lib/apt/lists/*
+
+      - uses: actions/checkout@v7
+
+      - name: Resolve the tag
+        id: tag
+        env:
+          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
+          TOKEN: ${{ gitea.token }}
+          INPUT_TAG: ${{ github.event.inputs.tag }}
+        run: |
+          set -eu
+          tag="$INPUT_TAG"
+          if [ -z "$tag" ]; then
+            tag="$(curl -sS -f -H "Authorization: token ${TOKEN}" "${API}/releases/latest" | jq -r '.tag_name')"
+          fi
+          case "$tag" in
+            v[0-9]*) ;;
+            *) echo "Tag '${tag}' is not a release tag (vX.Y.Z)."; exit 1 ;;
+          esac
+          echo "Mirroring ${tag}."
+          echo "tag=${tag}" >> "$GITHUB_OUTPUT"
+          echo "version=${tag#v}" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ steps.tag.outputs.tag }}
+          path: tag-tree
+
+      - name: Mirror the docs into the wiki
+        env:
+          VERSION: ${{ steps.tag.outputs.version }}
+          WIKI_TOKEN: ${{ gitea.token }}
+        run: |
+          set -eu
+          python3 .gitea/scripts/wiki_mirror.py --source tag-tree --out /tmp/wiki-out \
+            --version "$VERSION" --repo-url "${PUBLIC_BASE_URL}/${GITHUB_REPOSITORY}"
+          .gitea/scripts/wiki-push.sh /tmp/wiki-out "$VERSION"