Chemenu 2.1.0 - deterministischer Wissenskompiler

Chemenu kompiliert Rohnotizen zu einem verlinkten, quellengebundenen Wiki:
raw/ -> types/ + tools/ -> kb/ -> reports/. Was mechanisch ist, macht
tools/wikitool; was Urteil braucht, macht ein Agent unter Contracts, deren
Grenzen in Code durchgesetzt sind statt im Prompt.

Dieser Commit ist der Startpunkt der oeffentlichen Historie. Die vorherige
Entwicklung fand in einer privaten Instanz statt und ist nicht Teil dieses
Repositorys; ihre Erzaehlung steht vollstaendig in CHANGES.md, das mit 44
Eintraegen von 0.1.0 bis 2.1.0 erhalten geblieben ist.

Der mitgelieferte Korpus ist ein Testbett und eine Demo: 170 Seiten ueber den
Stack selbst - Gates, Lint, Versionierung, Suche, das Wiki-Muster. Er
dokumentiert das Werkzeug mit den eigenen Mitteln des Werkzeugs.

Lizenz: AGPL-3.0 fuer den Stack (tools/, types/), CC-BY-4.0 fuer die Inhalte.
Die Grenze zwischen beiden ist der Dateiplan, den dist export berechnet -
siehe NOTICE.

tools/chemenu/commands/lint.py

@@ -0,0 +1,470 @@
+"""Deterministic structural health checks for the wiki.
+
+This intentionally covers only what can be computed mechanically: broken
+wikilinks, orphan pages, index/page drift, frontmatter schema gaps, and
+filename/title mismatches. Semantic judgment (contradictions, staleness,
+what's worth writing about next) stays with the LLM - this report gives it a
+verified factual foundation instead of requiring it to re-derive these facts
+by reading every page.
+"""
+from __future__ import annotations
+
+import json
+from datetime import date
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from chemenu import config
+from chemenu.commands._util import rel_path, success
+from chemenu.frontmatter_io import frontmatter_error
+from chemenu.markdown_code import strip_code_spans
+from chemenu.provenance import broken_raw_refs as find_broken_raw_refs
+from chemenu.provenance import duplicate_raw_file_owners as find_duplicate_raw_file_owners
+from chemenu.provenance import extract_inline_cites
+from chemenu.provenance import legacy_citation_markers as find_legacy_citation_markers
+from chemenu.provenance import legacy_source_pages as find_legacy_source_pages
+from chemenu.provenance import orphan_footnote_defs as find_orphan_footnote_defs
+from chemenu.provenance import uncovered_raw_files as find_uncovered_raw_files
+from chemenu.provenance import undefined_footnote_refs as find_undefined_footnote_refs
+from chemenu.kb_scan import (
+    GENERATED_INDEX,
+    WIKILINK_RE,
+    build_link_graph,
+    find_duplicate_title_paths,
+    inbound_links,
+    load_kb_pages,
+)
+from chemenu.type_resolver import resolver
+
+# Style guide's one mechanically-checkable rule (hard oracle: a plain count).
+# The rest of the style guide (tone, AI-phrase avoidance) is a soft/proxy judgment
+# and stays with the LLM - see wiki-manage/wiki-ingest skill guidance, not lint.
+#
+# The unit is a quote, not a `>` line. It used to be the line, which measured
+# the wrap width the rule has no opinion about: one quotation written long
+# counted 1 and the same quotation wrapped at 100 columns counted 4. An author
+# who took the finding seriously made the page harder to read to quiet it.
+QUOTE_LIMIT = 2
+
+# How many hub pages `most_linked` reports. Purely informational (wiki-status
+# surfaces it); not a finding, so the cutoff only bounds report size.
+MOST_LINKED_COUNT = 10
+
+
+def count_quote_blocks(body: str) -> int:
+    """How many distinct blockquotes `body` carries.
+
+    A run of consecutive `>` lines is one quote; a blank line or any
+    non-quoted line ends it. Code is masked out first, so a `>` inside a
+    fenced shell transcript is a prompt, not a quotation.
+
+    Lazy continuation - a quote whose wrapped lines drop the `>` - reads here
+    as two quotes rather than one. That over-counts in the direction the limit
+    already errs on, and the corpus prefixes every line, so the alternative
+    (tracking paragraph state) buys nothing.
+    """
+    count, in_quote = 0, False
+    for line in strip_code_spans(body).splitlines():
+        is_quote = line.lstrip().startswith(">")
+        if is_quote and not in_quote:
+            count += 1
+        in_quote = is_quote
+    return count
+
+
+def run_lint(kb_dir: Path) -> dict:
+    pages = load_kb_pages(kb_dir)
+    duplicate_titles = find_duplicate_title_paths(kb_dir, config.ROOT)
+
+    # Pages whose frontmatter can't be parsed read back as `{}` everywhere
+    # else, which would let them slip past every frontmatter-driven check
+    # below with no finding at all - so they are detected explicitly.
+    frontmatter_errors = []
+    for title, page in sorted(pages.items()):
+        reason = frontmatter_error(page.path)
+        if reason is None and not page.frontmatter.get("type"):
+            reason = "missing `type:` field"
+        if reason is not None:
+            frontmatter_errors.append({"page": title, "error": reason})
+
+    graph = build_link_graph(pages)
+    broken_links = [
+        {"page": title, "target": target}
+        for title, targets in graph.items()
+        for target in sorted(targets)
+        if target not in pages
+    ]
+
+    inbound = inbound_links({t: v for t, v in graph.items() if t != "index"})
+    orphan_pages = sorted(
+        title
+        for title, sources in inbound.items()
+        if not sources
+        and title not in ("index", "log")
+        # comparison pages are not linked to by design; index.md is sufficient coverage
+        and pages[title].kind != "comparison"
+    )
+
+    # Same link graph, opposite end: the most-linked-to pages are the wiki's
+    # hubs. Reported (not judged) so `wiki-status` can show them without
+    # re-deriving the graph.
+    inbound_counts = {title: len(sources) for title, sources in inbound.items()}
+    most_linked = [
+        {"page": title, "inbound": count}
+        for title, count in sorted(inbound_counts.items(), key=lambda kv: (-kv[1], kv[0]))
+        if count > 0
+    ][:MOST_LINKED_COUNT]
+
+    # The catalog is sharded: `kb/index.md` is a map carrying counts and links,
+    # and the page rows live in a generated INDEX.md per collection/area. Both
+    # halves have to be read, or every page reads as missing from the index.
+    index_text = "".join(
+        path.read_text(encoding="utf-8")
+        for path in [kb_dir / "index.md", *sorted(kb_dir.rglob(GENERATED_INDEX))]
+        if path.exists()
+    )
+    index_links = {m.group(1).strip() for m in WIKILINK_RE.finditer(index_text)}
+    missing_from_index = sorted(set(pages) - index_links - {"index", "log"})
+    dangling_index_entries = sorted(index_links - set(pages))
+
+    title_mismatches = []
+    for title, page in sorted(pages.items()):
+        if page.kind not in ("entity", "concept"):
+            continue
+        h1 = page.h1_title
+        if h1 is not None and h1 != title:
+            title_mismatches.append({"page": title, "h1": h1})
+
+    unmarked_provenance = []
+    for title, page in sorted(pages.items()):
+        if page.kind not in ("entity", "concept"):
+            continue
+        sources_list = page.frontmatter.get("sources") or []
+        if not sources_list and page.frontmatter.get("provenance") != "general":
+            unmarked_provenance.append(title)
+
+    citation_frontmatter_drift = []
+    for title, page in sorted(pages.items()):
+        sources_list = set(page.frontmatter.get("sources") or [])
+        cited = {cited_title for cited_title, _file in extract_inline_cites(page.body)}
+        cited.discard(title)  # a source page citing itself for a specific file within it is not drift
+        for missing_source in sorted(cited - sources_list):
+            citation_frontmatter_drift.append({"page": title, "cited_but_not_in_sources": missing_source})
+
+    legacy_citation_markers = find_legacy_citation_markers(pages)
+    undefined_footnote_refs = find_undefined_footnote_refs(pages)
+    orphan_footnote_defs = find_orphan_footnote_defs(pages)
+
+    # The frontmatter half of the link graph. `broken_links` above only walks
+    # `[[wikilinks]]` in page *bodies*, so a `related:`/`sources:`/`entities:`
+    # entry naming a page that does not exist - a rename that was not
+    # propagated, a deleted page, or a URL pasted where a title belongs - used
+    # to pass every check. Which fields hold page titles is declared by each
+    # type-spec's `page_ref_fields:`, not hardcoded here.
+    dangling_frontmatter_refs = []
+    for title, page in sorted(pages.items()):
+        type_path = page.frontmatter.get("type")
+        if not type_path:
+            continue
+        try:
+            ref_fields = resolver.get_page_ref_fields(type_path, page.path)
+        except ValueError:
+            continue  # unresolvable type is already reported as type_resolution_errors
+        for field in ref_fields:
+            for target in page.frontmatter.get(field) or []:
+                if target not in pages:
+                    dangling_frontmatter_refs.append(
+                        {"page": title, "field": field, "target": target}
+                    )
+
+    quote_limit_violations = []
+    for title, page in sorted(pages.items()):
+        quote_count = count_quote_blocks(page.body)
+        if quote_count > QUOTE_LIMIT:
+            quote_limit_violations.append({"page": title, "quote_count": quote_count})
+
+    # Type system validation. Lint reports are not validated here: they are
+    # written to `reports/` outside kb/ and are never pages, so nothing this
+    # loop scans can be one.
+    invalid_type_paths = []
+    type_resolution_errors = []
+    schema_validation_errors = []
+
+    for title, page in sorted(pages.items()):
+        type_path = page.frontmatter.get("type")
+        if not type_path:
+            continue
+
+        # Check if type path is valid
+        if not type_path.endswith('.md'):
+            invalid_type_paths.append({"page": title, "type": type_path, "error": "Type path must end with .md"})
+            continue
+
+        # Try to resolve and validate the type
+        try:
+            resolver.load_type_spec(type_path, page.path)
+
+            # Try schema validation
+            try:
+                resolver.validate_frontmatter(page.frontmatter, type_path, page.path)
+            except ValueError as schema_error:
+                schema_validation_errors.append({"page": title, "type": type_path, "error": str(schema_error)})
+
+        except ValueError as resolution_error:
+            type_resolution_errors.append({"page": title, "type": type_path, "error": str(resolution_error)})
+
+    return {
+        "generated": date.today().isoformat(),
+        "page_count": len(pages),
+        "frontmatter_errors": frontmatter_errors,
+        "broken_links": broken_links,
+        "orphan_pages": orphan_pages,
+        "most_linked": most_linked,
+        "inbound_counts": inbound_counts,
+        "missing_from_index": missing_from_index,
+        "dangling_index_entries": dangling_index_entries,
+        "title_mismatches": title_mismatches,
+        "duplicate_titles": duplicate_titles,
+        "uncovered_raw_files": find_uncovered_raw_files(config.RAW_DIR, pages),
+        "broken_raw_refs": find_broken_raw_refs(pages),
+        "duplicate_raw_file_owners": find_duplicate_raw_file_owners(pages),
+        "legacy_source_pages": find_legacy_source_pages(pages),
+        "unmarked_provenance": unmarked_provenance,
+        "citation_frontmatter_drift": citation_frontmatter_drift,
+        "legacy_citation_markers": legacy_citation_markers,
+        "undefined_footnote_refs": undefined_footnote_refs,
+        "orphan_footnote_defs": orphan_footnote_defs,
+        "dangling_frontmatter_refs": dangling_frontmatter_refs,
+        "quote_limit_violations": quote_limit_violations,
+        "invalid_type_paths": invalid_type_paths,
+        "type_resolution_errors": type_resolution_errors,
+        "schema_validation_errors": schema_validation_errors,
+    }
+
+
+def _section(lines: list[str], title: str, items: list, formatter) -> None:
+    lines.append(f"## {title}")
+    lines.append("")
+    if not items:
+        lines.append("None found.")
+    else:
+        for item in items:
+            lines.append(f"- {formatter(item)}")
+    lines.append("")
+
+
+def render_markdown(report: dict) -> str:
+    lines = [f"# Structural Lint Report ({report['generated']})", ""]
+    lines.append(f"Scanned {report['page_count']} pages under `wiki/`. This report covers only")
+    lines.append("mechanically-verifiable structural issues; see the Semantic Review section")
+    lines.append("below for judgment calls the LLM should complete.")
+    lines.append("")
+
+    _section(
+        lines, "Unreadable Frontmatter", report["frontmatter_errors"],
+        lambda i: f"[[{i['page']}]] - {i['error']}",
+    )
+    _section(
+        lines, "Broken Wikilinks", report["broken_links"],
+        lambda i: f"[[{i['page']}]] links to missing [[{i['target']}]]",
+    )
+    _section(lines, "Orphan Pages (no inbound links)", report["orphan_pages"], lambda i: f"[[{i}]]")
+    _section(
+        lines, f"Most-Linked Pages (top {MOST_LINKED_COUNT} hubs)", report["most_linked"],
+        lambda i: f"[[{i['page']}]] - {i['inbound']} inbound link(s)",
+    )
+    _section(lines, "Pages Missing from index.md", report["missing_from_index"], lambda i: f"[[{i}]]")
+    _section(lines, "Dangling index.md Entries", report["dangling_index_entries"], lambda i: f"[[{i}]]")
+    _section(
+        lines, "Duplicate Titles (naming collisions)", report["duplicate_titles"],
+        lambda i: f"`{i['stem']}` -> {', '.join(f'`{p}`' for p in i['paths'])}",
+    )
+    _section(
+        lines, "Filename / H1 Title Mismatches", report["title_mismatches"],
+        lambda i: f"[[{i['page']}]] H1 is '{i['h1']}'",
+    )
+    _section(
+        lines, "Uncovered Raw Files (no source page)", report["uncovered_raw_files"],
+        lambda i: f"`{i}`",
+    )
+    _section(
+        lines, "Broken raw_files References", report["broken_raw_refs"],
+        lambda i: f"[[{i['page']}]] -> `{i['raw_path']}` (does not exist)",
+    )
+    _section(
+        lines, "Raw Files With More Than One Owner", report["duplicate_raw_file_owners"],
+        lambda i: f"`{i['raw_file']}` is claimed by " + ", ".join(f"[[{t}]]" for t in i["owners"]),
+    )
+    _section(
+        lines, "Legacy source: Field (not yet migrated to raw_files:)", report["legacy_source_pages"],
+        lambda i: f"[[{i['page']}]] source: `{i['source']}` ({i['reason']})",
+    )
+    _section(
+        lines, "Pages Missing provenance: general Marker", report["unmarked_provenance"],
+        lambda i: f"[[{i}]] has no sources and is not marked `provenance: general`",
+    )
+    _section(
+        lines, "Citation / Frontmatter Drift", report["citation_frontmatter_drift"],
+        lambda i: f"[[{i['page']}]] cites [[{i['cited_but_not_in_sources']}]] inline but it is missing from frontmatter `sources:`",
+    )
+    _section(
+        lines, "Legacy Citation Markers (pre-migration `^[[...]]`)", report["legacy_citation_markers"],
+        lambda i: f"[[{i['page']}]] still has `{i['marker']}` - run `wikitool cite add` and replace it with the `[^cite-id]` it prints",
+    )
+    _section(
+        lines, "Undefined Footnote References", report["undefined_footnote_refs"],
+        lambda i: f"[[{i['page']}]] references `[^{i['ref']}]`, which has no `[^{i['ref']}]: [[...]]` definition",
+    )
+    _section(
+        lines, "Orphan Footnote Definitions", report["orphan_footnote_defs"],
+        lambda i: f"[[{i['page']}]] defines `[^{i['id']}]` (-> [[{i['source']}]]) but nothing references it - run `wikitool cite sync`",
+    )
+    _section(
+        lines, "Dangling Frontmatter References", report["dangling_frontmatter_refs"],
+        lambda i: f"[[{i['page']}]] `{i['field']}:` names `{i['target']}`, which is not a page",
+    )
+    _section(
+        lines, "Invalid Type Paths", report["invalid_type_paths"],
+        lambda i: f"[[{i['page']}]] has type: `{i['type']}` - {i['error']}",
+    )
+    _section(
+        lines, "Type Resolution Errors", report["type_resolution_errors"],
+        lambda i: f"[[{i['page']}]] type: `{i['type']}` - {i['error']}",
+    )
+    _section(
+        lines, "Schema Validation Errors", report["schema_validation_errors"],
+        lambda i: f"[[{i['page']}]] type: `{i['type']}` - {i['error']}",
+    )
+    _section(
+        lines, f"Pages Exceeding Quote Limit (>{QUOTE_LIMIT}/page)", report["quote_limit_violations"],
+        lambda i: f"[[{i['page']}]] has {i['quote_count']} quotes - trim or confirm they're load-bearing",
+    )
+
+    lines.append("## Semantic Review (LLM to complete)")
+    lines.append("")
+    lines.append("- Contradictions across pages: TODO")
+    lines.append("- Stale claims (unconfirmed >6 months): TODO")
+    lines.append("- Suggested new pages / missing cross-references: TODO")
+    lines.append("")
+    return "\n".join(lines)
+
+
+# Sections that always carry content but are not findings, so the summary
+# handles them separately: a hub list is a statistic, and the semantic review
+# is the checklist that follows the report rather than part of it.
+INFORMATIONAL_SECTIONS = ("Most-Linked Pages",)
+SEMANTIC_REVIEW_SECTION = "Semantic Review"
+
+
+def _split_sections(markdown: str) -> tuple[str, list[tuple[str, str]]]:
+    """Cut a rendered report into its preamble and (title, body) sections."""
+    preamble, *rest = markdown.split("\n## ")
+    sections = []
+    for part in rest:
+        title, _, body = part.partition("\n")
+        sections.append((title.strip(), body.strip()))
+    return preamble.rstrip(), sections
+
+
+def render_summary(report: dict) -> str:
+    """The same report with the empty sections removed.
+
+    On a healthy corpus the full report is better than 90% "None found.", so
+    reading it in the terminal means paging past the answer. The file on disk
+    stays complete - this is what gets printed, and the written path underneath
+    it is how the rest is reached without running lint a second time.
+    """
+    preamble, sections = _split_sections(render_markdown(report))
+    findings, trailing = [], []
+    for title, body in sections:
+        if title.startswith(SEMANTIC_REVIEW_SECTION):
+            trailing.append((title, body))
+        elif body != "None found." and not title.startswith(INFORMATIONAL_SECTIONS):
+            findings.append((title, body))
+    lines = [preamble, ""]
+    if not findings:
+        lines += ["No structural findings.", ""]
+    for title, body in findings + trailing:
+        lines += [f"## {title}", "", body, ""]
+    return "\n".join(lines)
+
+
+def default_report_path(report: dict) -> Path:
+    """Where a report goes when the caller names no path.
+
+    `reports/` is derived and gitignored ([reports/CONTRACT.md]), so writing
+    here by default costs the tree nothing.
+    """
+    return config.ROOT / "reports" / f"Lint Report {report['generated']}.md"
+
+
+# Findings that make a tree structurally wrong rather than merely untidy.
+# `orphan_pages` is deliberately absent: many pages are validly reachable
+# through the index or navigation only. `quote_limit_violations` is advisory
+# too - it flags a habit, not a broken tree.
+#
+# One definition, used by `lint --fail-on-error` and by the eval scorecard: if
+# the two disagreed, a run could pass its score while lint refused it.
+HARD_ERROR_KEYS = (
+    "frontmatter_errors",
+    "broken_links",
+    "dangling_index_entries",
+    "duplicate_titles",
+    "broken_raw_refs",
+    "duplicate_raw_file_owners",
+    "legacy_source_pages",
+    "citation_frontmatter_drift",
+    "legacy_citation_markers",
+    "undefined_footnote_refs",
+    "orphan_footnote_defs",
+    "dangling_frontmatter_refs",
+    "invalid_type_paths",
+    "type_resolution_errors",
+    "schema_validation_errors",
+)
+
+
+def has_hard_errors(report: dict) -> bool:
+    return any(report.get(key) for key in HARD_ERROR_KEYS)
+
+
+def lint_command(
+    json_out: bool = typer.Option(False, "--json", help="Print the raw findings as JSON and write no report"),
+    markdown_out: Optional[Path] = typer.Option(None, "--markdown", help="Write the markdown report here instead of the default reports/Lint Report <date>.md"),
+    full: bool = typer.Option(False, "--full", help="Print the whole report instead of only the sections with findings"),
+    fail_on_error: bool = typer.Option(False, "--fail-on-error", help="Exit non-zero if hard errors were found"),
+):
+    """Run structural lint checks against kb/.
+
+    Unless `--json` is given, the full report is always written to a file and
+    its path is printed. That path is the point: a lint report is long, and an
+    agent that only saw it on stdout had no way back to the part it scrolled
+    past except by running lint again - two budget slots for one look at the
+    corpus.
+    """
+    report = run_lint(config.KB_DIR)
+
+    if json_out:
+        typer.echo(json.dumps(report, indent=2))
+        if fail_on_error and has_hard_errors(report):
+            raise typer.Exit(code=1)
+        return
+
+    typer.echo(render_markdown(report) if full else render_summary(report))
+
+    target = markdown_out or default_report_path(report)
+    frontmatter = (
+        "---\n"
+        "type: types/lint-report.md\n"
+        f"created: {report['generated']}\n"
+        f"summary: Structural lint report - {report['page_count']} pages scanned\n"
+        "---\n\n"
+    )
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(frontmatter + render_markdown(report) + "\n", encoding="utf-8")
+    success(f"Full report written to {rel_path(target)}")
+
+    if fail_on_error and has_hard_errors(report):
+        raise typer.Exit(code=1)