feat(workflow): Gitea-Wiki als generierter Spiegel von docs/ (#192)
Konverter .gitea/scripts/wiki_mirror.py, Push-Skript wiki-push.sh, Job wiki-mirror in release.yml, Dispatch-Workflow wiki-mirror.yml und --check-Schritt in ci.yml. Version 9.0.0-beta.13. Files changed: - .gitea/scripts/wiki-push.sh - .gitea/scripts/wiki_mirror.py - .gitea/workflows/ci.yml - .gitea/workflows/release.yml - .gitea/workflows/wiki-mirror.yml - CHANGES.md - README.md - VERSION - docs/dev/development.md - tools/chemenu/tests/test_wiki_mirror.py Co-Authored-By: Claude Code <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
This commit is contained in:
torbenandClaude Code committed
1 parent
45ae04cfee
commit
90ea0eea1b
10 files changed
+869
-3
No files matched your search
Executable
+293
@@ -0,0 +1,293 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convert the documentation of one release tree into the flat page set of a Gitea wiki.
|
||||
|
||||
wiki_mirror.py --source <tree> --out <dir> --version <X.Y.Z> [--repo-url <url>] [--check]
|
||||
|
||||
The output directory becomes the whole content of the wiki repository, so the conversion is
|
||||
deterministic: the same tree and version give the same bytes. `--check` converts in memory,
|
||||
writes nothing and fails only on a name collision or an unresolvable relative link.
|
||||
|
||||
Plain standard library on purpose: it runs in CI before any tool environment exists, and it
|
||||
must work against the tree of any past tag.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import sys
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
DEFAULT_REPO_URL = "https://gitea.nehmer.net/torben/chemenu"
|
||||
RESERVED_NAMES = {"home", "install", "_sidebar"}
|
||||
AREA_ORDER = ("tutorials", "how-to", "reference", "explanation")
|
||||
DEV_AREA = "dev"
|
||||
FLAT_GROUP = "Documentation"
|
||||
|
||||
_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:")
|
||||
_FENCE = re.compile(r"^\s*(`{3,}|~{3,})")
|
||||
_LINK = re.compile(r"""\]\(\s*([^)\s<][^)\s]*)((?:\s+(?:"[^"]*"|'[^']*'))?)\s*\)""")
|
||||
_H1 = re.compile(r"^# +(.+?)\s*#*\s*$")
|
||||
|
||||
|
||||
class Page:
|
||||
def __init__(self, path: str, name: str, segs: tuple[str, ...] | None, index: bool):
|
||||
self.path = path
|
||||
self.name = name
|
||||
self.segs = segs # sidebar position below docs/; None for Home and Install
|
||||
self.index = index # a README.md: stands for its directory
|
||||
|
||||
|
||||
def _capitalise(segment: str) -> str:
|
||||
return segment[:1].upper() + segment[1:]
|
||||
|
||||
|
||||
def collect_pages(source: str) -> tuple[list[Page], list[str]]:
|
||||
"""Every page to mirror, in a stable order, plus the collisions found among their names."""
|
||||
pages: list[Page] = []
|
||||
if os.path.isfile(os.path.join(source, "README.md")):
|
||||
pages.append(Page("README.md", "Home", None, False))
|
||||
if os.path.isfile(os.path.join(source, "INSTALL.md")):
|
||||
pages.append(Page("INSTALL.md", "Install", None, False))
|
||||
|
||||
docs = os.path.join(source, "docs")
|
||||
found: list[str] = []
|
||||
for base, dirs, files in os.walk(docs):
|
||||
dirs.sort()
|
||||
for file in sorted(files):
|
||||
if file.endswith(".md"):
|
||||
rel = os.path.relpath(os.path.join(base, file), source)
|
||||
found.append(rel.replace(os.sep, "/"))
|
||||
for path in sorted(found):
|
||||
parts = path[len("docs/"):-len(".md")].split("/")
|
||||
index = parts[-1] == "README"
|
||||
if index:
|
||||
parts = parts[:-1]
|
||||
name = "-".join(_capitalise(p) for p in parts) if parts else FLAT_GROUP
|
||||
pages.append(Page(path, name, tuple(parts), index))
|
||||
|
||||
errors: list[str] = []
|
||||
seen: dict[str, str] = {}
|
||||
for page in pages:
|
||||
key = page.name.lower()
|
||||
if key in RESERVED_NAMES and page.segs is not None:
|
||||
errors.append(f"{page.path}: page name '{page.name}' is reserved for a generated page")
|
||||
elif key in seen:
|
||||
errors.append(
|
||||
f"name collision on '{page.name}' (compared without case): {seen[key]} and {page.path}"
|
||||
)
|
||||
else:
|
||||
seen[key] = page.path
|
||||
return pages, errors
|
||||
|
||||
|
||||
def mask_code_spans(line: str) -> str:
|
||||
"""The line with every inline code span blanked to NULs, same length, so offsets still match."""
|
||||
out = list(line)
|
||||
n = len(line)
|
||||
i = 0
|
||||
while i < n:
|
||||
if line[i] != "`":
|
||||
i += 1
|
||||
continue
|
||||
j = i
|
||||
while j < n and line[j] == "`":
|
||||
j += 1
|
||||
run = j - i
|
||||
k = j
|
||||
end = -1
|
||||
while k < n:
|
||||
if line[k] != "`":
|
||||
k += 1
|
||||
continue
|
||||
m = k
|
||||
while m < n and line[m] == "`":
|
||||
m += 1
|
||||
if m - k == run:
|
||||
end = m
|
||||
break
|
||||
k = m
|
||||
if end < 0:
|
||||
i = j
|
||||
continue
|
||||
for p in range(i, end):
|
||||
out[p] = "\0"
|
||||
i = end
|
||||
return "".join(out)
|
||||
|
||||
|
||||
class Converter:
|
||||
def __init__(self, source: str, version: str, repo_url: str, pages: list[Page]):
|
||||
self.source = source
|
||||
self.version = version
|
||||
self.repo_url = repo_url.rstrip("/")
|
||||
self.by_path = {p.path: p for p in pages}
|
||||
self.errors: list[str] = []
|
||||
|
||||
def tag_url(self, path: str) -> str:
|
||||
base = f"{self.repo_url}/src/tag/v{self.version}"
|
||||
return base if path in ("", ".") else f"{base}/{quote(path, safe='/')}"
|
||||
|
||||
def resolve(self, src: str, target: str) -> str | None:
|
||||
"""The replacement for one relative link target, or None (and an error recorded)."""
|
||||
path, hash_, anchor = target.partition("#")
|
||||
suffix = hash_ + anchor
|
||||
path = unquote(path)
|
||||
base = "" if path.startswith("/") else posixpath.dirname(src)
|
||||
joined = posixpath.normpath(posixpath.join(base, path.lstrip("/")))
|
||||
if joined == ".." or joined.startswith("../"):
|
||||
self.errors.append(f"{src}: link '{target}' leaves the tree")
|
||||
return None
|
||||
page = self.by_path.get(joined)
|
||||
if page is None and os.path.isdir(os.path.join(self.source, joined)):
|
||||
index = "README.md" if joined == "." else f"{joined}/README.md"
|
||||
page = self.by_path.get(index)
|
||||
if page is not None:
|
||||
return page.name + suffix
|
||||
if os.path.exists(os.path.join(self.source, joined)):
|
||||
return self.tag_url(joined) + suffix
|
||||
self.errors.append(f"{src}: link '{target}' points at '{joined}', which does not exist in the tree")
|
||||
return None
|
||||
|
||||
def convert_line(self, src: str, line: str) -> str:
|
||||
masked = mask_code_spans(line)
|
||||
edits = []
|
||||
for match in _LINK.finditer(masked):
|
||||
target = match.group(1)
|
||||
if _SCHEME.match(target) or target.startswith(("#", "//")):
|
||||
continue
|
||||
replacement = self.resolve(src, line[match.start(1):match.end(1)])
|
||||
if replacement is not None:
|
||||
edits.append((match.start(1), match.end(1), replacement))
|
||||
for start, end, replacement in reversed(edits):
|
||||
line = line[:start] + replacement + line[end:]
|
||||
return line
|
||||
|
||||
def convert(self, src: str, text: str) -> str:
|
||||
return "\n".join(
|
||||
line if in_code else self.convert_line(src, line) for line, in_code in code_flags(text)
|
||||
)
|
||||
|
||||
|
||||
def code_flags(text: str):
|
||||
"""Each line of the text with whether it belongs to a fenced code block (fences included)."""
|
||||
fence = None
|
||||
for line in text.split("\n"):
|
||||
m = _FENCE.match(line)
|
||||
if fence is None:
|
||||
if m:
|
||||
fence = m.group(1)
|
||||
yield line, m is not None
|
||||
continue
|
||||
yield line, True
|
||||
if m and m.group(1)[0] == fence[0] and len(m.group(1)) >= len(fence) and not line.strip().strip(fence[0]):
|
||||
fence = None
|
||||
|
||||
|
||||
def title_of(text: str, fallback: str) -> str:
|
||||
for line, in_code in code_flags(text):
|
||||
h = None if in_code else _H1.match(line)
|
||||
if h:
|
||||
return h.group(1).replace("[", "\\[").replace("]", "\\]")
|
||||
return fallback
|
||||
|
||||
|
||||
def sidebar(pages: list[Page], titles: dict[str, str]) -> str:
|
||||
def link(page: Page) -> str:
|
||||
return f"[{titles[page.path]}]({page.name})"
|
||||
|
||||
lines = []
|
||||
home = next((p for p in pages if p.name == "Home" and p.segs is None), None)
|
||||
install = next((p for p in pages if p.name == "Install" and p.segs is None), None)
|
||||
for page in (home, install):
|
||||
if page is not None:
|
||||
lines.append(f"* {link(page)}")
|
||||
|
||||
docs = [p for p in pages if p.segs is not None]
|
||||
page_at = {p.segs: p for p in docs}
|
||||
flat = [p for p in docs if len(p.segs) == 0 or (len(p.segs) == 1 and not p.index)]
|
||||
tree = [p for p in docs if p not in flat]
|
||||
|
||||
children: dict[tuple[str, ...], set[tuple[str, ...]]] = {}
|
||||
for page in tree:
|
||||
for i in range(1, len(page.segs) + 1):
|
||||
children.setdefault(page.segs[: i - 1], set()).add(page.segs[:i])
|
||||
|
||||
def label(key: tuple[str, ...]) -> str:
|
||||
page = page_at.get(key)
|
||||
return link(page) if page else _capitalise(key[-1])
|
||||
|
||||
def render(key: tuple[str, ...], depth: int) -> None:
|
||||
lines.append(" " * depth + f"* {label(key)}")
|
||||
for child in sorted(children.get(key, ()), key=lambda k: k[-1].lower()):
|
||||
render(child, depth + 1)
|
||||
|
||||
if flat:
|
||||
lines.append(f"* {FLAT_GROUP}")
|
||||
for page in sorted(flat, key=lambda p: p.name.lower()):
|
||||
lines.append(f" * {link(page)}")
|
||||
|
||||
tops = children.get((), set())
|
||||
known = [(s,) for s in AREA_ORDER if (s,) in tops]
|
||||
rest = sorted((k for k in tops if k[0] not in AREA_ORDER and k[0] != DEV_AREA), key=lambda k: k[0].lower())
|
||||
last = [(DEV_AREA,)] if (DEV_AREA,) in tops else []
|
||||
for key in known + rest + last:
|
||||
render(key, 0)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def build(source: str, version: str, repo_url: str) -> tuple[dict[str, str], list[str]]:
|
||||
"""The wiki as {file name: content}, and every error found while building it."""
|
||||
pages, errors = collect_pages(source)
|
||||
converter = Converter(source, version, repo_url, pages)
|
||||
files: dict[str, str] = {}
|
||||
titles: dict[str, str] = {}
|
||||
for page in pages:
|
||||
with open(os.path.join(source, page.path), encoding="utf-8", newline="") as fh:
|
||||
text = fh.read()
|
||||
titles[page.path] = title_of(text, page.name)
|
||||
notice = (
|
||||
f"*Generated from [`{page.path}`]({converter.tag_url(page.path)}) at v{version}"
|
||||
" – edit the repository, not this wiki.*"
|
||||
)
|
||||
files[f"{page.name}.md"] = notice + "\n\n" + converter.convert(page.path, text)
|
||||
files["_Sidebar.md"] = sidebar(pages, titles)
|
||||
return files, errors + converter.errors
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--source", required=True, help="the tree to mirror")
|
||||
parser.add_argument("--out", help="empty or missing directory that receives the wiki content")
|
||||
parser.add_argument("--version", required=True, help="X.Y.Z of the tree; names the tag links point at")
|
||||
parser.add_argument("--repo-url", default=DEFAULT_REPO_URL)
|
||||
parser.add_argument("--check", action="store_true", help="convert, write nothing, fail on a problem")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if not args.check and not args.out:
|
||||
parser.error("--out is required unless --check is given")
|
||||
if not os.path.isdir(args.source):
|
||||
print(f"ERROR --source {args.source} is not a directory", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
files, errors = build(args.source, args.version, args.repo_url)
|
||||
if errors:
|
||||
for error in errors:
|
||||
print(f"ERROR {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if not args.check:
|
||||
if os.path.isdir(args.out) and os.listdir(args.out):
|
||||
print(f"ERROR --out {args.out} is not empty", file=sys.stderr)
|
||||
return 1
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
for name, content in files.items():
|
||||
with open(os.path.join(args.out, name), "w", encoding="utf-8", newline="") as fh:
|
||||
fh.write(content)
|
||||
print(f"{len(files)} wiki files for v{args.version}" + (" (checked, nothing written)" if args.check else ""))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user