chemenu/tools/chemenu/commands/index_build.py

"""Deterministically regenerate the wiki's catalog from every page's frontmatter.

This replaces manual statistics counting and manual sorted-row insertion, which
was a repeated source of errors (miscounts, wrong alphabetical position) when
done by hand.

The catalog is **sharded**, not one file. `kb/index.md` is a map: statistics,
one row per collection and per area, and a link to the shard that lists those
pages. The tables themselves live in a generated `INDEX.md` inside each
collection, and an area that grows past `SHARD_THRESHOLD` rows gets its own.

Why: a single flat catalog has to be read in full to answer any question about
it, so its cost grows with the wiki while the answer being looked for does not.
At a few hundred pages that is tens of thousands of tokens spent to learn three
filenames. The wiki's own `Index Scaling` page sets the threshold used here.
The map stays small enough to browse; `wikitool search` answers everything else.
"""
from __future__ import annotations

import re
from dataclasses import dataclass, field
from datetime import date
from pathlib import Path

import typer

from chemenu import config
from chemenu.commands._util import rel_path, success
from chemenu.kb_collections import iter_kb_collections
from chemenu.page import Page
from chemenu.kb_scan import GENERATED_INDEX, load_kb_pages
from chemenu.type_resolver import resolver

app = typer.Typer(help="Manage the generated wiki catalog (kb/index.md + per-collection INDEX.md).")

TABLE_HEADER = "| Page | Type | Summary | Last Modified |"
TABLE_SEP = "|------|------|---------|----------------|"

SUMMARY_HEADINGS = ("Description", "Definition", "Summary")

# Rows per area before it is split into its own shard. From the wiki's own
# `Index Scaling` page ("split table sections at >50 entries"), kept as a plain
# number so growth is handled by arithmetic rather than by a judgment call.
SHARD_THRESHOLD = 50

# Display title for pages sitting directly in a collection root rather than in
# an area subdirectory.
UNGROUPED_TITLE = "All"

DO_NOT_EDIT = "<!-- Generated by `wikitool index rebuild`. Do not hand-edit. -->"


def _summary(page: Page) -> str:
    fm_summary = page.frontmatter.get("summary")
    if fm_summary:
        return str(fm_summary).strip()
    for heading in SUMMARY_HEADINGS:
        match = re.search(rf"^## {heading}\s*\n+(.+)", page.body, re.MULTILINE)
        if match:
            line = match.group(1).strip().splitlines()[0].strip()
            if line and not line.upper().startswith("TODO"):
                return line[:117] + "..." if len(line) > 120 else line
    return "TODO: add summary"


def _last_modified(page: Page) -> str:
    for key in ("modified", "date", "created"):
        value = page.frontmatter.get(key)
        if value:
            return str(value)
    return date.fromtimestamp(page.path.stat().st_mtime).isoformat()


def _type_label(page: Page) -> str:
    return page.subtype or page.kind or "unknown"


def _table(pages: list[Page]) -> list[str]:
    lines = [TABLE_HEADER, TABLE_SEP]
    for page in sorted(pages, key=lambda p: p.title.lower()):
        lines.append(
            f"| [[{page.title}]] | {_type_label(page)} | {_summary(page)} | {_last_modified(page)} |"
        )
    return lines


def _anchor(title: str) -> str:
    """GitHub-style heading anchor, so the map can deep-link into a shard."""
    slug = re.sub(r"[^a-z0-9\s-]", "", title.lower())
    return re.sub(r"\s+", "-", slug.strip())


@dataclass
class Area:
    """One grouping inside a collection: a subdirectory, or the collection root
    for pages that sit directly in it."""

    name: str
    title: str
    pages: list[Page] = field(default_factory=list)
    own_shard: bool = False

    @property
    def count(self) -> int:
        return len(self.pages)


@dataclass
class Collection:
    name: str
    areas: list[Area] = field(default_factory=list)

    @property
    def count(self) -> int:
        return sum(area.count for area in self.areas)


def _area_titles() -> dict[str, str]:
    """Display titles for entity areas, taken from the entity type-spec's own
    `layout:` rather than a hardcoded map - so a new subtype names its own
    section by adding a type-spec, with no code change."""
    layout = resolver.get_layout(resolver.find_type_by_name("entity")) or {}
    return {spec.get("dir", key): spec.get("title", key.title()) for key, spec in layout.items()}


def group_pages(kb_dir: Path, pages: dict[str, Page]) -> list[Collection]:
    """Group pages by their physical location: collection directory, then area
    subdirectory.

    Location rather than `kind` because a shard lives in the directory it
    describes, and the two agree by construction: a type-spec's `base_dir:` is
    what put the page there.
    """
    titles = _area_titles()
    grouped: dict[str, dict[str, Area]] = {}

    # Seed from the collections that exist on disk, not only from the ones that
    # happen to hold pages: an empty collection is a real (if unfilled) part of
    # the wiki, and dropping it from the map would hide it from every reader.
    for collection_dir in iter_kb_collections(kb_dir):
        grouped.setdefault(collection_dir.name, {})

    for page in sorted(pages.values(), key=lambda p: p.title.lower()):
        try:
            parts = page.path.relative_to(kb_dir).parts
        except ValueError:  # pragma: no cover - pages always live under kb_dir
            continue
        if len(parts) < 2:
            collection_name, area_name = "(kb root)", ""
        else:
            collection_name = parts[0]
            area_name = parts[1] if len(parts) > 2 else ""
        areas = grouped.setdefault(collection_name, {})
        area = areas.get(area_name)
        if area is None:
            title = titles.get(area_name, area_name.title()) if area_name else UNGROUPED_TITLE
            area = Area(name=area_name, title=title)
            areas[area_name] = area
        area.pages.append(page)

    collections = []
    for name in sorted(grouped):
        ordered = sorted(grouped[name].values(), key=lambda a: (a.name == "", a.title.lower()))
        for area in ordered:
            area.own_shard = bool(area.name) and area.count > SHARD_THRESHOLD
        collections.append(Collection(name=name, areas=ordered))
    return collections


def build_area_shard(area: Area) -> str:
    lines = [DO_NOT_EDIT, "", f"# {area.title}", "", f"{area.count} page(s).", ""]
    lines.extend(_table(area.pages))
    lines.append("")
    return "\n".join(lines) + "\n"


def build_collection_shard(collection: Collection) -> str:
    lines = [DO_NOT_EDIT, "", f"# kb/{collection.name}/ - Index", ""]
    lines.append(f"{collection.count} page(s). Regenerated by `wikitool index rebuild`.")
    lines.append("")
    for area in collection.areas:
        lines.append(f"## {area.title}")
        lines.append("")
        if area.own_shard:
            lines.append(
                f"{area.count} page(s) - listed in "
                f"[{area.name}/{GENERATED_INDEX}]({area.name}/{GENERATED_INDEX})."
            )
        else:
            lines.extend(_table(area.pages))
        lines.append("")
    return "\n".join(lines) + "\n"


def build_index_map(collections: list[Collection]) -> str:
    """The root catalog: counts and pointers, no page rows.

    Deliberately carries no summaries. A summary is what makes a hit worth
    opening, and that judgment belongs where the hit is produced - `search` and
    the shards - not in a file every reader pays for in full.
    """
    totals = {c.name: c.count for c in collections}
    total = sum(totals.values())

    lines = [
        DO_NOT_EDIT,
        "",
        "# Wiki Index",
        "",
        "A map of the wiki, not a catalog of it: counts and pointers only.",
        "",
        "To *find* a page, search instead of reading this file:",
        "",
        '- `tools/wikitool search "<text>"` - ranked text search, with summaries',
        "- `tools/wikitool search --field entity_type=system --field 'confidence<0.6'`"
        " - structured query over frontmatter",
        "",
        "The page tables live in a generated `INDEX.md` inside each collection, linked below.",
        "",
        "## Statistics",
        "",
        f"- **Total Pages:** {total}",
    ]
    for name in sorted(totals):
        lines.append(f"- **{name.title()}:** {totals[name]}")
    lines.append(f"- **Last Updated:** {date.today().isoformat()}")
    lines.append("")
    lines.append("---")
    lines.append("")
    lines.append("## Collections")
    lines.append("")
    lines.append("| Collection | Pages | Index |")
    lines.append("|------------|------:|-------|")
    for collection in collections:
        target = f"{collection.name}/{GENERATED_INDEX}"
        lines.append(f"| `{collection.name}/` | {collection.count} | [{target}]({target}) |")
    lines.append("")

    for collection in collections:
        listed = [area for area in collection.areas if area.name]
        if not listed:
            continue
        lines.append(f"### {collection.name}/")
        lines.append("")
        lines.append("| Area | Pages | Index |")
        lines.append("|------|------:|-------|")
        for area in listed:
            if area.own_shard:
                target = f"{collection.name}/{area.name}/{GENERATED_INDEX}"
            else:
                target = f"{collection.name}/{GENERATED_INDEX}#{_anchor(area.title)}"
            lines.append(f"| {area.title} | {area.count} | [{target}]({target}) |")
        lines.append("")

    lines.append("---")
    lines.append("")
    lines.append("## Notes")
    lines.append("")
    lines.append(
        "This map and every `INDEX.md` under `kb/` are generated by "
        "`wikitool index rebuild`. Do not hand-edit them."
    )
    lines.append("")
    lines.append("To add a new page, run `wikitool new ...`, then `wikitool index rebuild`.")
    return "\n".join(lines) + "\n"


def plan_index(kb_dir: Path) -> dict[Path, str]:
    """Every file the catalog consists of, as {path: content}.

    Returning the whole plan instead of writing as it goes is what makes the
    stale-shard sweep possible: anything named `INDEX.md` that is not in the
    plan is a leftover from a collection or area that no longer exists.
    """
    collections = group_pages(kb_dir, load_kb_pages(kb_dir))
    plan: dict[Path, str] = {kb_dir / "index.md": build_index_map(collections)}
    for collection in collections:
        collection_dir = kb_dir / collection.name
        if not collection_dir.is_dir():
            continue
        plan[collection_dir / GENERATED_INDEX] = build_collection_shard(collection)
        for area in collection.areas:
            if area.own_shard:
                plan[collection_dir / area.name / GENERATED_INDEX] = build_area_shard(area)
    return plan


def stale_shards(kb_dir: Path, plan: dict[Path, str]) -> list[Path]:
    """Generated shards on disk that the current plan does not produce."""
    return sorted(p for p in kb_dir.rglob(GENERATED_INDEX) if p not in plan)


def build_index(kb_dir: Path) -> str:
    """The root map. Kept as a named function because callers (and tests) ask
    for "the index" meaning the entry point, not the whole plan."""
    return build_index_map(group_pages(kb_dir, load_kb_pages(kb_dir)))


@app.command("rebuild")
def index_rebuild(
    dry_run: bool = typer.Option(
        False, "--dry-run", help="Print what would be written instead of writing it"
    ),
):
    plan = plan_index(config.KB_DIR)
    stale = stale_shards(config.KB_DIR, plan)

    if dry_run:
        for path in sorted(plan):
            typer.echo(f"--- {rel_path(path)}")
            # nl=False: the content already ends in a newline.
            typer.echo(plan[path], nl=False)
        for path in stale:
            typer.echo(f"--- would remove stale shard: {rel_path(path)}")
        return

    for path, content in plan.items():
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_text(content, encoding="utf-8")
    for path in stale:
        path.unlink()

    shards = len(plan) - 1
    removed = f", removed {len(stale)} stale" if stale else ""
    success(f"Rebuilt {rel_path(config.INDEX_FILE)} and {shards} shard(s){removed}")