chemenu/.gitea/scripts/wiki_mirror.py

#!/usr/bin/env python3
"""Convert the documentation of one release tree into the flat page set of a Gitea wiki.

    wiki_mirror.py --source <tree> --out <dir> --version <X.Y.Z> [--repo-url <url>] [--check]

The output directory becomes the whole content of the wiki repository, so the conversion is
deterministic: the same tree and version give the same bytes. `--check` converts in memory,
writes nothing and fails only on a name collision or an unresolvable relative link.

Plain standard library on purpose: it runs in CI before any tool environment exists, and it
must work against the tree of any past tag.
"""
from __future__ import annotations

import argparse
import os
import posixpath
import re
import sys
from urllib.parse import quote, unquote

DEFAULT_REPO_URL = "https://gitea.nehmer.net/torben/chemenu"
RESERVED_NAMES = {"home", "install", "_sidebar"}
AREA_ORDER = ("tutorials", "how-to", "reference", "explanation")
DEV_AREA = "dev"
FLAT_GROUP = "Documentation"

_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:")
_FENCE = re.compile(r"^\s*(`{3,}|~{3,})")
_LINK = re.compile(r"""\]\(\s*([^)\s<][^)\s]*)((?:\s+(?:"[^"]*"|'[^']*'))?)\s*\)""")
_H1 = re.compile(r"^# +(.+?)\s*#*\s*$")


class Page:
    def __init__(self, path: str, name: str, segs: tuple[str, ...] | None, index: bool):
        self.path = path
        self.name = name
        self.segs = segs  # sidebar position below docs/; None for Home and Install
        self.index = index  # a README.md: stands for its directory


def _capitalise(segment: str) -> str:
    return segment[:1].upper() + segment[1:]


def collect_pages(source: str) -> tuple[list[Page], list[str]]:
    """Every page to mirror, in a stable order, plus the collisions found among their names."""
    pages: list[Page] = []
    if os.path.isfile(os.path.join(source, "README.md")):
        pages.append(Page("README.md", "Home", None, False))
    if os.path.isfile(os.path.join(source, "INSTALL.md")):
        pages.append(Page("INSTALL.md", "Install", None, False))

    docs = os.path.join(source, "docs")
    found: list[str] = []
    for base, dirs, files in os.walk(docs):
        dirs.sort()
        for file in sorted(files):
            if file.endswith(".md"):
                rel = os.path.relpath(os.path.join(base, file), source)
                found.append(rel.replace(os.sep, "/"))
    for path in sorted(found):
        parts = path[len("docs/"):-len(".md")].split("/")
        index = parts[-1] == "README"
        if index:
            parts = parts[:-1]
        name = "-".join(_capitalise(p) for p in parts) if parts else FLAT_GROUP
        pages.append(Page(path, name, tuple(parts), index))

    errors: list[str] = []
    seen: dict[str, str] = {}
    for page in pages:
        key = page.name.lower()
        if key in RESERVED_NAMES and page.segs is not None:
            errors.append(f"{page.path}: page name '{page.name}' is reserved for a generated page")
        elif key in seen:
            errors.append(
                f"name collision on '{page.name}' (compared without case): {seen[key]} and {page.path}"
            )
        else:
            seen[key] = page.path
    return pages, errors


def mask_code_spans(line: str) -> str:
    """The line with every inline code span blanked to NULs, same length, so offsets still match."""
    out = list(line)
    n = len(line)
    i = 0
    while i < n:
        if line[i] != "`":
            i += 1
            continue
        j = i
        while j < n and line[j] == "`":
            j += 1
        run = j - i
        k = j
        end = -1
        while k < n:
            if line[k] != "`":
                k += 1
                continue
            m = k
            while m < n and line[m] == "`":
                m += 1
            if m - k == run:
                end = m
                break
            k = m
        if end < 0:
            i = j
            continue
        for p in range(i, end):
            out[p] = "\0"
        i = end
    return "".join(out)


class Converter:
    def __init__(self, source: str, version: str, repo_url: str, pages: list[Page]):
        self.source = source
        self.version = version
        self.repo_url = repo_url.rstrip("/")
        self.by_path = {p.path: p for p in pages}
        self.errors: list[str] = []

    def tag_url(self, path: str) -> str:
        base = f"{self.repo_url}/src/tag/v{self.version}"
        return base if path in ("", ".") else f"{base}/{quote(path, safe='/')}"

    def resolve(self, src: str, target: str) -> str | None:
        """The replacement for one relative link target, or None (and an error recorded)."""
        path, hash_, anchor = target.partition("#")
        suffix = hash_ + anchor
        path = unquote(path)
        base = "" if path.startswith("/") else posixpath.dirname(src)
        joined = posixpath.normpath(posixpath.join(base, path.lstrip("/")))
        if joined == ".." or joined.startswith("../"):
            self.errors.append(f"{src}: link '{target}' leaves the tree")
            return None
        page = self.by_path.get(joined)
        if page is None and os.path.isdir(os.path.join(self.source, joined)):
            index = "README.md" if joined == "." else f"{joined}/README.md"
            page = self.by_path.get(index)
        if page is not None:
            return page.name + suffix
        if os.path.exists(os.path.join(self.source, joined)):
            return self.tag_url(joined) + suffix
        self.errors.append(f"{src}: link '{target}' points at '{joined}', which does not exist in the tree")
        return None

    def convert_line(self, src: str, line: str) -> str:
        masked = mask_code_spans(line)
        edits = []
        for match in _LINK.finditer(masked):
            target = match.group(1)
            if _SCHEME.match(target) or target.startswith(("#", "//")):
                continue
            replacement = self.resolve(src, line[match.start(1):match.end(1)])
            if replacement is not None:
                edits.append((match.start(1), match.end(1), replacement))
        for start, end, replacement in reversed(edits):
            line = line[:start] + replacement + line[end:]
        return line

    def convert(self, src: str, text: str) -> str:
        return "\n".join(
            line if in_code else self.convert_line(src, line) for line, in_code in code_flags(text)
        )


def code_flags(text: str):
    """Each line of the text with whether it belongs to a fenced code block (fences included)."""
    fence = None
    for line in text.split("\n"):
        m = _FENCE.match(line)
        if fence is None:
            if m:
                fence = m.group(1)
            yield line, m is not None
            continue
        yield line, True
        if m and m.group(1)[0] == fence[0] and len(m.group(1)) >= len(fence) and not line.strip().strip(fence[0]):
            fence = None


def title_of(text: str, fallback: str) -> str:
    for line, in_code in code_flags(text):
        h = None if in_code else _H1.match(line)
        if h:
            return h.group(1).replace("[", "\\[").replace("]", "\\]")
    return fallback


def sidebar(pages: list[Page], titles: dict[str, str]) -> str:
    def link(page: Page) -> str:
        return f"[{titles[page.path]}]({page.name})"

    lines = []
    home = next((p for p in pages if p.name == "Home" and p.segs is None), None)
    install = next((p for p in pages if p.name == "Install" and p.segs is None), None)
    for page in (home, install):
        if page is not None:
            lines.append(f"* {link(page)}")

    docs = [p for p in pages if p.segs is not None]
    page_at = {p.segs: p for p in docs}
    flat = [p for p in docs if len(p.segs) == 0 or (len(p.segs) == 1 and not p.index)]
    tree = [p for p in docs if p not in flat]

    children: dict[tuple[str, ...], set[tuple[str, ...]]] = {}
    for page in tree:
        for i in range(1, len(page.segs) + 1):
            children.setdefault(page.segs[: i - 1], set()).add(page.segs[:i])

    def label(key: tuple[str, ...]) -> str:
        page = page_at.get(key)
        return link(page) if page else _capitalise(key[-1])

    def render(key: tuple[str, ...], depth: int) -> None:
        lines.append("  " * depth + f"* {label(key)}")
        for child in sorted(children.get(key, ()), key=lambda k: k[-1].lower()):
            render(child, depth + 1)

    if flat:
        lines.append(f"* {FLAT_GROUP}")
        for page in sorted(flat, key=lambda p: p.name.lower()):
            lines.append(f"  * {link(page)}")

    tops = children.get((), set())
    known = [(s,) for s in AREA_ORDER if (s,) in tops]
    rest = sorted((k for k in tops if k[0] not in AREA_ORDER and k[0] != DEV_AREA), key=lambda k: k[0].lower())
    last = [(DEV_AREA,)] if (DEV_AREA,) in tops else []
    for key in known + rest + last:
        render(key, 0)
    return "\n".join(lines) + "\n"


def build(source: str, version: str, repo_url: str) -> tuple[dict[str, str], list[str]]:
    """The wiki as {file name: content}, and every error found while building it."""
    pages, errors = collect_pages(source)
    converter = Converter(source, version, repo_url, pages)
    files: dict[str, str] = {}
    titles: dict[str, str] = {}
    for page in pages:
        with open(os.path.join(source, page.path), encoding="utf-8", newline="") as fh:
            text = fh.read()
        titles[page.path] = title_of(text, page.name)
        notice = (
            f"*Generated from [`{page.path}`]({converter.tag_url(page.path)}) at v{version}"
            " – edit the repository, not this wiki.*"
        )
        files[f"{page.name}.md"] = notice + "\n\n" + converter.convert(page.path, text)
    files["_Sidebar.md"] = sidebar(pages, titles)
    return files, errors + converter.errors


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
    parser.add_argument("--source", required=True, help="the tree to mirror")
    parser.add_argument("--out", help="empty or missing directory that receives the wiki content")
    parser.add_argument("--version", required=True, help="X.Y.Z of the tree; names the tag links point at")
    parser.add_argument("--repo-url", default=DEFAULT_REPO_URL)
    parser.add_argument("--check", action="store_true", help="convert, write nothing, fail on a problem")
    args = parser.parse_args(argv)

    if not args.check and not args.out:
        parser.error("--out is required unless --check is given")
    if not os.path.isdir(args.source):
        print(f"ERROR --source {args.source} is not a directory", file=sys.stderr)
        return 1

    files, errors = build(args.source, args.version, args.repo_url)
    if errors:
        for error in errors:
            print(f"ERROR {error}", file=sys.stderr)
        return 1

    if not args.check:
        if os.path.isdir(args.out) and os.listdir(args.out):
            print(f"ERROR --out {args.out} is not empty", file=sys.stderr)
            return 1
        os.makedirs(args.out, exist_ok=True)
        for name, content in files.items():
            with open(os.path.join(args.out, name), "w", encoding="utf-8", newline="") as fh:
                fh.write(content)
    print(f"{len(files)} wiki files for v{args.version}" + (" (checked, nothing written)" if args.check else ""))
    return 0


if __name__ == "__main__":
    sys.exit(main())