chemenu/tools/chemenu/catalog.py

"""How the corpus groups into collections and areas, with no CLI attached.

Split out of `commands/index_build.py` for the reason `lint_core.py` gives at
the top of itself: this is a pure function over a corpus directory, and it was
sitting in a module that imports `typer` and `rich`. `lint` needs the same
grouping - it is what answers "does this collection have areas, and is it over
the threshold?" (Gitea #59) - and `chemenu.api`, the read surface, may not
reach a command module at all. Importing it from there would have pulled the
whole CLI head in behind it.

So the split runs along the same line as lint's: everything that decides *how
the corpus is shaped* lives here; everything that decides *what the catalog
looks like* - the tables, the map, the shard files - stays in
`commands/index_build.py`, which imports from here.
"""
from __future__ import annotations

import re
from dataclasses import dataclass, field
from pathlib import Path

from chemenu.kb_collections import iter_kb_collections
from chemenu.page import Page
from chemenu.type_resolver import resolver

# Rows per area before it is split into its own shard. From the wiki's own
# `Index Scaling` page ("split table sections at >50 entries"), kept as a plain
# number so growth is handled by arithmetic rather than by a judgment call.
SHARD_THRESHOLD = 50

# The name `group_pages` gives the pseudo-collection of pages sitting directly
# under `kb/`, which no collection directory holds.
ROOT_COLLECTION = "(kb root)"

# Display title for pages sitting directly in a collection root rather than in
# an area subdirectory.
UNGROUPED_TITLE = "All"


def anchor(title: str) -> str:
    """GitHub-style heading anchor, so a map can deep-link into a shard.

    `\\w` rather than `a-z0-9`, which is not cosmetic: an area title follows the
    KB language, and the first non-English one (`Abläufe`) had its umlaut
    *deleted* rather than kept, so the map linked at `#ablufe` and the anchor it
    was aiming at was `#abläufe`. Every deep link into a shard whose title
    carries a non-ASCII letter was silently dead. Nothing surfaced it while the
    only areas were entity ones, whose titles happen to be ASCII throughout.
    """
    slug = re.sub(r"[^\w\s-]", "", title.lower(), flags=re.UNICODE)
    return re.sub(r"\s+", "-", slug.strip())


def type_label(page: Page) -> str:
    return page.subtype or page.kind or "unknown"


@dataclass
class Area:
    """One grouping inside a collection: a subdirectory, or the collection root
    for pages that sit directly in it."""

    name: str
    title: str
    pages: list[Page] = field(default_factory=list)
    own_shard: bool = False

    @property
    def count(self) -> int:
        return len(self.pages)


@dataclass
class Collection:
    name: str
    areas: list[Area] = field(default_factory=list)

    @property
    def count(self) -> int:
        return sum(area.count for area in self.areas)


def area_titles() -> dict[str, dict[str, str]]:
    """Display titles per collection: `{collection: {area_dir: title}}`, taken
    from each type-spec's own `layout:` rather than a hardcoded map - so a new
    subtype names its own section by adding a type-spec, with no code change.

    Every type-spec is read, not just `entity`'s. That hardcoding was the
    asymmetry behind Gitea #59: the axis a collection splits along is declared
    in `layout:`, and a second type declaring one would have had its areas
    titled by `.title()` on the directory name while entity's got their real
    names.

    Keyed by collection rather than by directory name alone, because two types
    writing into two collections may legitimately use the same area name for
    different things (`kb/entities/tools/` and a hypothetical
    `kb/concepts/tools/`); a flat map would hand the second one the first's
    title. The collection key is the type's `base_dir:`, which is what put the
    page in that directory to begin with.
    """
    titles: dict[str, dict[str, str]] = {}
    for type_path, _frontmatter in resolver.list_type_specs():
        try:
            if resolver.get_root(type_path) != "kb":
                continue
            base_dir = resolver.get_base_dir(type_path)
            layout = resolver.get_layout(type_path)
        except (ValueError, OSError):
            continue
        if not base_dir or not layout:
            continue
        per_collection = titles.setdefault(str(base_dir).strip("/"), {})
        for key, spec in layout.items():
            per_collection.setdefault(spec.get("dir", key), spec.get("title", str(key).title()))
    return titles


def group_pages(kb_dir: Path, pages: dict[str, Page]) -> list[Collection]:
    """Group pages by their physical location: collection directory, then area
    subdirectory.

    Location rather than `kind` because a shard lives in the directory it
    describes, and the two agree by construction: a type-spec's `base_dir:` is
    what put the page there.
    """
    titles = area_titles()
    grouped: dict[str, dict[str, Area]] = {}

    # Seed from the collections that exist on disk, not only from the ones that
    # happen to hold pages: an empty collection is a real (if unfilled) part of
    # the wiki, and dropping it from the map would hide it from every reader.
    for collection_dir in iter_kb_collections(kb_dir):
        grouped.setdefault(collection_dir.name, {})

    for page in sorted(pages.values(), key=lambda p: p.title.lower()):
        try:
            parts = page.path.relative_to(kb_dir).parts
        except ValueError:  # pragma: no cover - pages always live under kb_dir
            continue
        if len(parts) < 2:
            collection_name, area_name = ROOT_COLLECTION, ""
        else:
            collection_name = parts[0]
            area_name = parts[1] if len(parts) > 2 else ""
        areas = grouped.setdefault(collection_name, {})
        area = areas.get(area_name)
        if area is None:
            title = (
                titles.get(collection_name, {}).get(area_name, area_name.title())
                if area_name
                else UNGROUPED_TITLE
            )
            area = Area(name=area_name, title=title)
            areas[area_name] = area
        area.pages.append(page)

    collections = []
    for name in sorted(grouped):
        ordered = sorted(grouped[name].values(), key=lambda a: (a.name == "", a.title.lower()))
        for area in ordered:
            area.own_shard = bool(area.name) and area.count > SHARD_THRESHOLD
        collections.append(Collection(name=name, areas=ordered))
    return collections