"""Read/write markdown files with YAML frontmatter, matching the formatting conventions already used across kb/ (inline flow-style lists, unquoted dates). We deliberately avoid a generic yaml.dump() for the frontmatter block because PyYAML's default block-style output does not match the existing convention (e.g. `tags: [a, b, c]` on one line). Instead we serialize each top-level key explicitly, preserving dict insertion order. """ from __future__ import annotations import datetime import re from pathlib import Path from typing import Any import yaml from chemenu.errors import ValidationError try: # pragma: no cover - which branch runs depends on the host's libyaml from yaml import CSafeLoader as _Loader except ImportError: # pragma: no cover from yaml import SafeLoader as _Loader FRONTMATTER_RE = re.compile(r"\A---\n(.*?)\n---\n?(.*)\Z", re.DOTALL) # Frontmatter is a flat block of scalars and one-line lists. Real pages sit # well under a kilobyte, so this is not a budget anyone writes against - it is # the bound that keeps parse cost proportional to the corpus rather than to # whatever a single file claims to be. It matters because the parser is on the # read path, and the read path is what gets exposed. MAX_FRONTMATTER_BYTES = 64 * 1024 # YAML anchors and aliases are refused outright rather than budgeted. A page's # frontmatter has no use for them, and alias expansion is where a small # document becomes an enormous object graph: 267 bytes of nested aliases # compose in 0.2 ms into 672,603 nodes, growing 9**n with nesting depth at # constant parse time. A size limit alone does not touch that, because the # input stays small - see `ALIAS_BOMB` in the tests. # # The check runs on the *event* stream (`yaml.parse`), which is streaming and # resolves nothing - so asking the question costs O(text) and never triggers # the expansion it is asking about. `"*"` is a necessary character in any alias # node, so its absence proves absence without parsing at all, which is the case # every real page takes. _ALIAS_HINT = "*" class FrontmatterError(ValidationError): """Frontmatter that cannot be used: missing, malformed, oversized, or refused by a limit. Raised only by the strict entry points - the permissive ones report it as a string instead.""" def _load_frontmatter(fm_text: str) -> tuple[dict[str, Any] | None, str | None]: """Parse one frontmatter block into (mapping, error). Exactly one is None. The single parser behind both `read_page()` and `frontmatter_error()`. They used to have one each, which is how the permissive path could degrade to `{}` for a reason the strict path described differently - and how every caller wanting both answers read and parsed the file twice. """ encoded = len(fm_text.encode("utf-8")) if encoded > MAX_FRONTMATTER_BYTES: return None, ( f"frontmatter is {encoded} bytes, over the {MAX_FRONTMATTER_BYTES}-byte limit" ) if _ALIAS_HINT in fm_text: try: for event in yaml.parse(fm_text, Loader=_Loader): if isinstance(event, yaml.AliasEvent): return None, ( "frontmatter uses a YAML alias (`*" f"{event.anchor}`); anchors and aliases are not allowed here" ) except yaml.YAMLError as exc: return None, f"invalid YAML frontmatter: {_first_line(exc)}" except RecursionError: return None, "frontmatter is nested too deeply to parse" try: parsed = yaml.load(fm_text, Loader=_Loader) except yaml.YAMLError as exc: return None, f"invalid YAML frontmatter: {_first_line(exc)}" except RecursionError: # PyYAML composes recursively, so deep nesting exhausts the interpreter # stack rather than raising a YAMLError. Unbounded nesting is bounded by # MAX_FRONTMATTER_BYTES; this catches what fits under it. return None, "frontmatter is nested too deeply to parse" if parsed is None: return None, "empty frontmatter block" if not isinstance(parsed, dict): return None, f"frontmatter is {type(parsed).__name__}, expected a mapping" return parsed, None def _first_line(exc: Exception) -> str: text = str(exc) return text.splitlines()[0] if text else exc.__class__.__name__ def read_page_with_error(path: Path) -> tuple[dict[str, Any], str, str | None]: """(frontmatter, body, error) - the permissive read, with the reason it was permissive handed back instead of dropped. A page whose YAML is broken reads as `{}`, and a `{}` page then has no `sources` and no `kind`: it drops out of `--field '!sources'` - precisely the query meant to find pages in bad shape - while looking to the caller like a page that simply did not match. Returning the reason is what lets a caller say so instead of losing the page quietly. """ text = path.read_text(encoding="utf-8") match = FRONTMATTER_RE.match(text) if not match: return {}, text, "no `---` frontmatter block" fm_text, body = match.group(1), match.group(2) parsed, error = _load_frontmatter(fm_text) if error == "empty frontmatter block": # An empty block is a legitimate shape for the permissive read - it # carries no fields, and there is nothing to lose. Only `lint` treats # it as a finding. return {}, body, error return (parsed or {}), body, error def read_page(path: Path) -> tuple[dict[str, Any], str]: """Return (frontmatter_dict, body) for a markdown file. If the file has no frontmatter block, returns ({}, full_text). Deliberately permissive: malformed YAML degrades to an empty dict so bulk operations never crash on one bad page. Use `read_page_with_error()` (search does) or `frontmatter_error()` (`wikitool lint` does) to surface those pages instead of losing them silently. """ frontmatter, body, _ = read_page_with_error(path) return frontmatter, body def read_page_strict(path: Path) -> tuple[dict[str, Any], str]: """`read_page()` that raises `FrontmatterError` instead of degrading. For any path that ingests frontmatter this instance did not write itself. The permissive read is right for bulk operations over a corpus the operator committed; it is wrong the moment the document arrives from outside, where "unparseable" must stop the document rather than empty it. """ frontmatter, body, error = read_page_with_error(path) if error is not None: raise FrontmatterError(f"{path}: {error}") return frontmatter, body def frontmatter_error(path: Path) -> str | None: """Return a human-readable reason why `path`'s frontmatter can't be used, or None if it parses into a dict. This is the strict counterpart to `read_page()`: without it, a page whose YAML is malformed (or whose frontmatter block is missing entirely) reads back as `{}` and then quietly slips past every frontmatter-driven check. """ return read_page_with_error(path)[2] def _format_scalar(value: Any, flow: bool = False) -> str: """Render one frontmatter value. `flow` says it is going inside a `[...]` sequence, where more characters are indicators than at document level.""" if value is None: return '""' if isinstance(value, bool): return "true" if value else "false" if isinstance(value, (int, float)): # Bare, not routed through the string-quoting logic below: that logic # asks "would this text read back as the same *string*", which is the # wrong question for a value that was never a string - a real float # like 0.9 would fail that probe (it reads back as a float) and get # wrongly quoted into the string "0.9", silently changing its type on # the next read. return str(value) if isinstance(value, (datetime.date, datetime.datetime)): return value.isoformat() text = str(value) needs_quoting = ( text == "" or text[0] in "[{'\"#&*!|>%@`" or text.strip() != text or ": " in text or not _round_trips_as_string(text, flow=flow) ) if needs_quoting: return _quote(text) return text def _quote(text: str) -> str: """Render `text` as a quoted YAML scalar valid in any context. The dumper is asked for a one-element flow sequence and the brackets are taken off again, rather than a bare scalar: a bare plain scalar comes back with a `...` document-end marker attached, which is correct YAML for a document and nonsense inside a `[...]` list. Going through the library either way keeps the escaping rules where they belong - the same reason `_round_trips_as_string` asks the loader instead of listing cases. """ dumped = yaml.safe_dump([text], default_flow_style=True, width=10**9).strip() return dumped[1:-1].strip() def _round_trips_as_string(text: str, flow: bool = False) -> bool: """Whether writing `text` unquoted would read back as the same string. A string that merely *looks* like another YAML type comes back as that type: `"1945"` written bare is an int on the next read, and a schema expecting a string then rejects a page nothing visibly changed. The same holds for floats, YAML 1.1's `yes`/`no`/`on`/`off` booleans, `null`, and forms like `0x1F` or `1_000`. Asking the loader instead of listing the cases is deliberate: the reader and the writer then agree by construction, and a resolver rule this code never heard of cannot drift out from under it. Dates need no exception here. A date field holds a `datetime.date`, which `_format_scalar` renders bare before ever reaching this function - see `normalize_dates` for the other half of that contract. `flow` asks the question in the context list values are actually written in. At document level a comma is an ordinary character, so the probe says "safe to write bare"; inside the `[...]` this file emits for every list it is an indicator, and the value silently comes back as two elements. That is how a `raw_files:` entry naming a file with a comma in its name lost half of itself on write - after `--set` had parsed it correctly. """ probe, expected = (f"[{text}]", [text]) if flow else (text, text) try: return yaml.load(probe, Loader=_Loader) == expected except yaml.YAMLError: # Unparseable bare - quoting is exactly the fix. return False def normalize_dates(frontmatter: dict[str, Any]) -> dict[str, Any]: """A copy of `frontmatter` with every `datetime.date` rendered as an ISO string, at the top level and one list deep. Frontmatter carries dates as `datetime.date` - that is what `yaml.safe_load` produces for a bare `2026-08-29`, what `_format_scalar` writes back bare, and therefore what the whole corpus stores. The schemas nonetheless declare those fields `type: string`, so anything validating raw frontmatter has to convert first. This lives here rather than beside one validator because there are two, and only one of them used to do it: `TypeResolver.validate_frontmatter` (behind `lint`) normalized, while `touch`'s `validate_fields` did not. The gap was invisible only because `touch` happened to write date *strings*. """ normalized: dict[str, Any] = {} for key, value in frontmatter.items(): if isinstance(value, datetime.date): normalized[key] = value.isoformat() elif isinstance(value, list): normalized[key] = [ item.isoformat() if isinstance(item, datetime.date) else item for item in value ] else: normalized[key] = value return normalized def _format_list(items: list[Any]) -> str: if not items: return "[]" return "[" + ", ".join(_format_scalar(v, flow=True) for v in items) + "]" def _is_single_key_mapping(value: Any) -> bool: return isinstance(value, dict) and len(value) == 1 def _format_mapping_list(key: str, items: list[Any]) -> str: """A list holding `label: target` pairs, rendered block-style. The inline `[...]` form this file uses everywhere else cannot carry a mapping without quoting rules nobody reading the file would guess, so a labelled edge list is the one place block style earns its keep: related: - depends-on: Hermes - Borealis Bare strings mixed in stay bare - that is an edge whose label has not been declared yet, and promoting it to some default here would erase exactly what `lint` is looking for. """ lines = [f"{key}:"] for item in items: if _is_single_key_mapping(item): (label, target), = item.items() lines.append(f" - {_format_scalar(label)}: {_format_scalar(target)}") else: lines.append(f" - {_format_scalar(item)}") return "\n".join(lines) def dump_frontmatter(frontmatter: dict[str, Any]) -> str: lines = [] for key, value in frontmatter.items(): if isinstance(value, list) and any(_is_single_key_mapping(v) for v in value): lines.append(_format_mapping_list(key, value)) elif isinstance(value, list): lines.append(f"{key}: {_format_list(value)}") else: lines.append(f"{key}: {_format_scalar(value)}") return "\n".join(lines) def write_page(path: Path, frontmatter: dict[str, Any], body: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) fm_text = dump_frontmatter(frontmatter) if not body.startswith("\n"): body = "\n" + body content = f"---\n{fm_text}\n---{body}" if not content.endswith("\n"): content += "\n" path.write_text(content, encoding="utf-8", newline="\n")