chemenu/tools/chemenu/kb_collections.py

"""Discover the collections under kb/ from the filesystem.

The repo's structural rule is that a directory under `kb/` is a collection
exactly when it contains a `COLLECTION.md`. Making that the *only* definition -
rather than a list of directory names somewhere in code or docs - is what lets
`mkdir kb/<name> && $EDITOR kb/<name>/COLLECTION.md` add a collection without a
code change, and what keeps the check honest when someone adds a directory and
forgets the contract.

Two corollaries are enforced rather than documented:

  * A `COLLECTION.md` nested inside a collection is invalid. Subdirectories of a
    collection are *areas*: they inherit the enclosing contract, so a second
    contract below it would create two answers to "which rules apply here?".
  * A `COLLECTION.md` outside `kb/` is invalid. `raw/`, `types/` and `reports/`
    are pipeline stages, not collections; they carry a README or a root
    type-spec instead. Without this check the word "collection" quietly widens
    back out to "any directory with a contract in it".
"""
from __future__ import annotations

from pathlib import Path
from typing import Any

from chemenu import config

CONTRACT_NAME = "COLLECTION.md"

# What a collection declares about itself, in `COLLECTION.md`'s frontmatter.
#
# Presence on the filesystem says a collection *exists*; it cannot say who owns
# the rules inside it. A `COLLECTION.md` is instance-owned - the distribution
# ships a `.template` per default collection and the instance writes the real
# one - so the two facts the stack still needs from it have to be declared
# rather than inferred from the directory name, which an instance is free to
# choose.
PROFILE_FIELD = "profile"
REQUIRED_BY_STACK_FIELD = "required_by_stack"

# Which link labels a page in this collection may use, per destination
# collection. The **source** collection decides, which is the whole point: an
# edge is an authored reader-aid written on the page that asserts it, so the
# rules that govern it are the rules of the collection that page lives in. A
# destination is another collection's name, or `any`.
#
# This is Commonplace's ADR-019 adopted directly, and it is what makes a
# 35-label catalogue usable: a collection authorises the six that make sense
# from it, and the rest of the palette is simply not on its menu.
OUTBOUND_FIELD = "outbound"
ANY_DESTINATION = "any"

# The types `wikitool` itself depends on existing, as opposed to ones an
# instance keeps because they are useful. `source` is here because the whole
# `raw/ -> kb/` provenance path is built on it: `sources coverage` asks which
# raw files no source page claims, every `[^cite-id]` resolves to a source page,
# and `sources rebuild-index` writes `kb/provenance.md` from them. All three ask
# `page.kind == "source"`, so what is load-bearing is the type-spec's `name:`
# and its schema requiring `raw_files:` - not the directory, not the title
# prefix, and not a word of its prose or its template.
#
# That is the whole anchor, and it is deliberately this small: the four page
# type-specs belong to the instance (see types/type-spec.md), so anything more
# would be the stack reaching into a file it does not own.
STACK_REQUIRED_TYPES = ("source",)
STACK_REQUIRED_TYPE_FIELDS = {"source": ("raw_files",)}


def stack_required_collections() -> tuple[str, ...]:
    """Collection names an instance may not rename or drop.

    **Derived, not listed.** The required collection is whichever one the
    required type writes into - so an instance that legitimately renames
    `kb/sources/` to something else, and says so in the type-spec's `base_dir:`,
    stays consistent instead of tripping a constant that hardcoded the old name.
    A second literal list would only be a copy that drifts.
    """
    from chemenu.type_resolver import resolver

    names: list[str] = []
    for type_name in STACK_REQUIRED_TYPES:
        try:
            type_path = resolver.find_type_by_name(type_name)
            if not type_path:
                continue
            if resolver.get_root(type_path) != "kb":
                continue
            base_dir = resolver.get_base_dir(type_path)
        except (ValueError, OSError):
            continue
        if base_dir:
            names.append(str(base_dir).strip("/"))
    return tuple(dict.fromkeys(names))


def iter_kb_collections(kb_dir: Path | None = None) -> list[Path]:
    """Return every collection directory under kb/, sorted by name.

    A collection is an immediate child directory of `kb/` containing a
    `COLLECTION.md`. Nested contracts are deliberately not returned - they are
    invalid, and `stray_collection_contracts()` reports them.
    """
    root = kb_dir if kb_dir is not None else config.KB_DIR
    if not root.is_dir():
        return []
    return sorted(
        (child for child in root.iterdir() if child.is_dir() and (child / CONTRACT_NAME).is_file()),
        key=lambda path: path.name,
    )


def kb_collection_of(path: Path, kb_dir: Path | None = None) -> Path | None:
    """Return the collection a path belongs to, or None if it is outside kb/.

    Areas resolve to their enclosing collection, so
    `kb/entities/systems/hermes.md` answers `kb/entities`.
    """
    root = kb_dir if kb_dir is not None else config.KB_DIR
    try:
        relative = path.resolve().relative_to(root.resolve())
    except ValueError:
        return None
    if not relative.parts:
        return None
    candidate = root / relative.parts[0]
    return candidate if candidate.is_dir() and (candidate / CONTRACT_NAME).is_file() else None


def stray_collection_contracts(root: Path | None = None, kb_dir: Path | None = None) -> list[Path]:
    """Return every misplaced COLLECTION.md, sorted.

    Misplaced means either nested inside a collection (an area may not carry its
    own contract) or located anywhere outside `kb/`. `commonplace/` is skipped:
    it is a vendored, read-only knowledge base with its own collection tree and
    is not governed by this repo's layout.
    """
    repo_root = root if root is not None else config.ROOT
    collections_root = kb_dir if kb_dir is not None else config.KB_DIR
    collections = {path.resolve() for path in iter_kb_collections(collections_root)}

    stray: list[Path] = []
    for contract in repo_root.rglob(CONTRACT_NAME):
        if _is_vendored(contract, repo_root):
            continue
        parent = contract.parent.resolve()
        if parent in collections:
            continue
        stray.append(contract)
    return sorted(stray)


def collection_declaration(collection: Path) -> dict[str, Any]:
    """A collection's own `COLLECTION.md` frontmatter, or `{}` if it has none.

    Permissive like every other frontmatter read in this package: an unreadable
    declaration degrades to empty here and is reported by `docs verify`, rather
    than taking down the discovery every command starts with.
    """
    from chemenu.frontmatter_io import read_page

    contract = collection / CONTRACT_NAME
    if not contract.is_file():
        return {}
    frontmatter, _ = read_page(contract)
    return frontmatter


def authorised_labels(source: str, destination: str, kb_dir: Path | None = None) -> set[str]:
    """Labels a page in `source` may use on an edge into `destination`.

    The union of the destination's own entry and `any`. An empty result means
    the collection authorises nothing for that destination - which is a real
    answer ("do not link there from here"), not a missing declaration.
    """
    root = kb_dir if kb_dir is not None else config.KB_DIR
    declared = collection_declaration(root / source).get(OUTBOUND_FIELD)
    if not isinstance(declared, dict):
        return set()
    labels: set[str] = set()
    for key in (destination, ANY_DESTINATION):
        entry = declared.get(key)
        if isinstance(entry, list):
            labels.update(str(label).strip() for label in entry if str(label).strip())
    return labels


LABELLED_EDGE_FIELD = "related"


def collections_that_can_carry_labelled_edges() -> set[str]:
    """Collection names whose offered page types actually have somewhere to put
    a labelled edge.

    Derived from `page_ref_fields:`, the same way `stack_required_collections()`
    is derived from `base_dir:`: a type that does not offer `related:` cannot
    carry a label, no matter what its collection's contract authorises.
    """
    from chemenu.type_resolver import resolver

    names: set[str] = set()
    for type_path, _frontmatter in resolver.list_type_specs():
        try:
            if resolver.get_root(type_path) != "kb":
                continue
            base_dir = resolver.get_base_dir(type_path)
            fields = resolver.get_page_ref_fields(type_path)
        except (ValueError, OSError):
            continue
        if base_dir and LABELLED_EDGE_FIELD in fields:
            names.add(str(base_dir).strip("/"))
    return names


def declaration_issues(kb_dir: Path | None = None) -> list[str]:
    """What each `COLLECTION.md` fails to declare about itself.

    Two fields, for two questions the filesystem cannot answer. `profile:`
    names the entry in `instructions/kb-profiles.md` this collection adopted -
    free text, because the profile catalogue is a palette rather than an enum,
    and a collection an instance invented has no entry there to name.
    `required_by_stack:` is not the instance's to choose at all: it must agree
    with what `stack_required_collections()` derives from the required types, so
    a collection whose contract claims the stack depends on it - or one the
    stack does depend on and that says it does not - is a finding rather than a
    preference.
    """
    root = kb_dir if kb_dir is not None else config.KB_DIR
    issues: list[str] = []

    required = stack_required_collections()
    can_label = collections_that_can_carry_labelled_edges()
    present = {path.name for path in iter_kb_collections(root)}
    for name in required:
        if name not in present:
            issues.append(
                f"kb/{name}/ is missing - it is where the stack-required `source` type writes, "
                f"and `sources coverage`, `[^cite-id]` resolution and `kb/provenance.md` all "
                f"depend on those pages existing"
            )

    for collection in iter_kb_collections(root):
        relative = f"kb/{collection.name}/{CONTRACT_NAME}"
        declared = collection_declaration(collection)
        if not declared:
            issues.append(
                f"{relative} has no frontmatter - it must declare `{PROFILE_FIELD}:` and "
                f"`{REQUIRED_BY_STACK_FIELD}:` (see instructions/kb-profiles.md)"
            )
            continue

        profile = declared.get(PROFILE_FIELD)
        if not isinstance(profile, str) or not profile.strip():
            issues.append(
                f"{relative}: `{PROFILE_FIELD}:` is missing or empty - name the profile from "
                f"instructions/kb-profiles.md this collection adopted, or `none`"
            )

        required_flag = declared.get(REQUIRED_BY_STACK_FIELD)
        expected = collection.name in required
        if not isinstance(required_flag, bool):
            issues.append(
                f"{relative}: `{REQUIRED_BY_STACK_FIELD}:` is missing or not a boolean - "
                f"it must be {str(expected).lower()} for this collection"
            )
        elif required_flag != expected:
            issues.append(
                f"{relative}: `{REQUIRED_BY_STACK_FIELD}: {str(required_flag).lower()}` "
                f"contradicts the stack, which "
                + (
                    "does depend on this collection by name"
                    if expected
                    else "depends on no collection of this name"
                )
                + f" - it must be {str(expected).lower()}"
            )

        # An `outbound:` block on a collection whose types offer no `related:`
        # authorises labels that no page there can write. That is not a harmless
        # extra: it reads as a licence, so the label gets written into the prose
        # by hand instead - an identifier back in free text, which is the exact
        # thing the labelled-edge model exists to end. The two halves have to
        # move together, so the check names both directions of the fix.
        if declared.get(OUTBOUND_FIELD) and collection.name not in can_label:
            issues.append(
                f"{relative}: `{OUTBOUND_FIELD}:` authorises labels, but no page type writing "
                f"into kb/{collection.name}/ offers a `{LABELLED_EDGE_FIELD}:` field - so no "
                f"page here can carry a labelled edge. Either drop the block, or give the "
                f"type-spec a `{LABELLED_EDGE_FIELD}:` in its `page_ref_fields:` and schema"
            )
    return issues


def _is_vendored(path: Path, repo_root: Path) -> bool:
    try:
        relative = path.relative_to(repo_root)
    except ValueError:
        return True
    return relative.parts[:1] == ("commonplace",)