tools: raw_cmd.py split into the package commands/raw/, one module per subcommand (#206)

Pure restructuring: every raw subcommand's output, help text and exit code is unchanged; the shared helpers sit in raw/_common.py.

Files changed:
- CHANGES.md
- VERSION
- tools/README.md
- tools/chemenu/cli.py
- tools/chemenu/commands/raw/__init__.py
- tools/chemenu/commands/raw/_common.py
- tools/chemenu/commands/raw/accept.py
- tools/chemenu/commands/raw/capture.py
- tools/chemenu/commands/raw/derive.py
- tools/chemenu/commands/raw/fetch.py
- tools/chemenu/commands/raw/pending.py
- tools/chemenu/commands/raw_cmd.py
- tools/chemenu/tests/test_docs_verify.py
- tools/chemenu/tests/test_export_guidelines.py
- tools/chemenu/tests/test_raw_capture.py
- tools/chemenu/tests/test_raw_cmd.py
- tools/chemenu/tests/test_raw_derive.py
- tools/chemenu/tests/test_raw_derive_testfiles.py
- tools/chemenu/tests/test_raw_fetch.py
- tools/chemenu/upload.py

Co-Authored-By: Claude Code <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2

tools/README.md

@@ -119,7 +119,7 @@ tools/
     guideline_export.py    `export guidelines`' core: select pages by `search` predicates, render them into one deterministic GUIDELINES.md, and recognise an export again (`EXPORT_MARKER`) - with no CLI attached
     search/                pluggable search backends, plus service.py - the search core
     tasks/                 the task-tracker provider layer: protocol.py (TaskReader/TaskWriter), config.py (.wikitool-tasks.json), one module per adapter - no instruction ever learns which provider it is
-    commands/              one module per command or command group: the terminal adapters
+    commands/              one module per command or command group: the terminal adapters - except `raw/`, a package with one module per subcommand and the helpers several of them share in `_common.py`
     tests/                 pytest suite - origin repository only, `dist export` leaves it out
 ```
 

tools/chemenu/cli.py

@@ -42,7 +42,7 @@ try:
         new_page,
         page_ops,
         provenance_cmd,
-        raw_cmd,
+        raw,
         review_cmd,
         run_budget,
         search as search_module,
@@ -167,7 +167,7 @@ app.add_typer(links_cmd.app, name="links")
 app.add_typer(index_build.app, name="index")
 app.add_typer(log_append.app, name="log")
 app.add_typer(provenance_cmd.app, name="sources")
-app.add_typer(raw_cmd.app, name="raw")
+app.add_typer(raw.app, name="raw")
 app.add_typer(upload_cmd.app, name="upload")
 app.add_typer(instructions_cmd.app, name="instructions")
 app.add_typer(run_budget.app, name="budget")

tools/chemenu/commands/raw/__init__.py

@@ -0,0 +1,19 @@
+"""`wikitool raw` - bring raw material into `incoming/`, and promote it from there
+into `raw/`.
+
+One module per subcommand; what two of them share is in `_common`. The Typer
+group is assembled here, and the modules are imported in the order the
+subcommands are listed in `-h` and in the generated command reference.
+"""
+import typer
+
+from chemenu.commands.raw import accept, pending, fetch, derive, capture
+
+app = typer.Typer(help="Bring raw material into incoming/, and promote it from there into raw/.")
+
+app.command("accept")(accept.raw_accept_command)
+app.command("pending")(pending.raw_pending_command)
+app.command("fetch")(fetch.raw_fetch_command)
+app.command("derive")(derive.raw_derive_command)
+app.command("capture")(capture.raw_capture_command)
+app.command("status")(capture.raw_status_command)

tools/chemenu/commands/raw/_common.py

@@ -0,0 +1,499 @@
+"""Helpers that more than one `wikitool raw` subcommand uses: the `incoming/`
+directory and its refusals, the date shard under `raw/`, the stems already taken,
+and the planning of file and folder moves. A helper only one subcommand uses
+lives in that subcommand's module.
+"""
+from __future__ import annotations
+
+import datetime
+import os
+import re
+from pathlib import Path
+from typing import Optional
+
+from rich.markup import escape
+
+from chemenu import config, filters, guideline_export, repo_capture
+from chemenu.commands._util import fail, path_budget_problem_for, rel_path
+from chemenu.errors import ValidationError
+from chemenu.provenance import citing_pages, source_pages_by_raw_file
+from chemenu.type_resolver import resolver
+
+
+# The type this command always promotes into eventually - hardcoded rather
+# than derived from --page, because there is no page yet in the common case
+# (see module docstring): the follow-up hint and the capture-field/enum
+# validation below both need *a* type to read from, and `source` is the only
+# one `raw accept` has ever scaffolded towards.
+_SOURCE_TYPE_PATH = "types/source.md"
+
+
+def _incoming_dir() -> Path:
+    return config.ROOT / "incoming"
+
+
+def _resolve(raw: Path) -> Path:
+    return raw if raw.is_absolute() else config.ROOT / raw
+
+
+def _shard_dir(today: Optional[datetime.date] = None) -> Path:
+    """`raw/<YYYY>/<MM>/`, computed from the accept date - never from a count,
+    so it can never rebalance and break a `[^cite-id]` anchor (Gitea #67).
+    """
+    d = today or datetime.date.today()
+    return config.RAW_DIR / f"{d.year:04d}" / f"{d.month:02d}"
+
+
+class _Refused(Exception):
+    """A check that runs before anything moves has failed; the message is what
+    `fail()` prints. Raised rather than failing on the spot so `raw pending`
+    can ask the same checks of every candidate without exiting - one set of
+    checks, two callers (Gitea #112)."""
+
+
+def _refusals_fail(fn, *args):
+    try:
+        return fn(*args)
+    except _Refused as exc:
+        fail(escape(str(exc)))
+
+
+def _check_directly_in_incoming(path: Path, incoming: Path) -> None:
+    """`path` must sit directly in `incoming/`.
+
+    Up to Gitea #112 one subdirectory level was tolerated and ignored, so an
+    old `incoming/<type>/` habit kept working after #67 stopped reading the
+    type from it. A subdirectory is a source of its own now - accepted as a
+    whole - so a file inside one is refused with both ways out named.
+    """
+    try:
+        rel = path.relative_to(incoming)
+    except ValueError:
+        raise _Refused(
+            f"{rel_path(path)} is not under incoming/ - `raw accept` and `raw fetch --html` "
+            "only take files from there. See raw/CONTRACT.md."
+        ) from None
+    if len(rel.parts) < 1:
+        raise _Refused(f"incoming/{rel.as_posix()} names no file.")
+    if len(rel.parts) > 1:
+        top = rel.parts[0]
+        raise _Refused(
+            f"incoming/{rel.as_posix()} lies in a subdirectory of incoming/ - a file is taken "
+            "only from directly inside incoming/, and a subdirectory is a source of its own. "
+            f"Either accept the whole folder as one source (raw accept incoming/{top}), or move "
+            "the file up into incoming/ and accept it there. See raw/CONTRACT.md."
+        )
+
+
+def _validate_under_incoming(path: Path, incoming: Path) -> None:
+    _refusals_fail(_check_directly_in_incoming, path, incoming)
+
+
+_YEAR_DIR = re.compile(r"\d{4}")
+
+
+def _occupied_stems(raw_dir: Path) -> dict[str, Path]:
+    """Every name occupied anywhere under `raw/`: file stems and bundle
+    directory names alike, at whichever level directly holds files.
+
+    Distinguishes the pre-#67 flat layout (`raw/<name>/<file-or-bundle>`)
+    from the #67 date shard (`raw/<YYYY>/<MM>/<file-or-bundle>`) structurally,
+    by whether a top-level directory name is a 4-digit year - not from a
+    hardcoded list of legacy type names, so this keeps working unchanged if a
+    legacy directory is ever renamed by hand.
+    """
+    occupied: dict[str, Path] = {}
+    if not raw_dir.is_dir():
+        return occupied
+    for top in raw_dir.iterdir():
+        if not top.is_dir():
+            continue  # raw/CONTRACT.md
+        if _YEAR_DIR.fullmatch(top.name):
+            for month in top.iterdir():
+                if month.is_dir():
+                    occupied.update(_leaf_group_entries(month))
+        else:
+            occupied.update(_leaf_group_entries(top))
+    return occupied
+
+
+def _leaf_group_entries(group_dir: Path) -> dict[str, Path]:
+    """The addressable entries directly inside one leaf-group directory
+    (a legacy type dir, or one `raw/<YYYY>/<MM>/`): a lone file occupies its
+    stem, a bundle directory occupies its own name."""
+    return {entry.stem if entry.is_file() else entry.name: entry for entry in group_dir.iterdir()}
+
+
+def _stem_collision_message(claimed_name: str, holder: Path) -> str:
+    example_target = holder
+    if holder.is_dir():
+        children = sorted(holder.iterdir())
+        example_target = children[0] if children else holder
+    return (
+        f'{rel_path(holder)} already claims the stem "{claimed_name}" under raw/.\n'
+        "  These are two different intents and only you can tell them apart:\n"
+        f"    Same source, new edition   -> tools/wikitool raw accept "
+        f"--replaces {rel_path(example_target)} <incoming file>\n"
+        "    A second, separate source  -> rename it in incoming/ (add a distinguishing "
+        "suffix) and accept it normally\n"
+        "  raw accept does not guess which one this is."
+    )
+
+
+def _check_files(resolved: list[Path]) -> None:
+    incoming = _incoming_dir()
+    for path in resolved:
+        if not path.is_file():
+            raise _Refused(f"{rel_path(path)} does not exist or is not a file.")
+        _check_directly_in_incoming(path, incoming)
+        if path.name == repo_capture.MANIFEST_NAME:
+            raise _Refused(_reserved_manifest_message(path))
+    _check_no_export(resolved)
+    names = [path.name for path in resolved]
+    if len(names) != len(set(names)):
+        raise _Refused("Two files share a filename; rename one before promoting.")
+
+
+def _check_no_export(paths: list[Path]) -> None:
+    """Refuse a file a wiki exported (`export guidelines`): it is generated
+    from `kb/`, so taking it into `raw/` would feed the wiki its own output
+    back as a source. `raw capture` already leaves such a file out; this
+    closes the way in through `incoming/`."""
+    exports = [p for p in paths if guideline_export.file_is_export(p)]
+    if exports:
+        listed = "\n".join(f"  - {rel_path(p)}" for p in exports)
+        raise _Refused(
+            "These files are a wiki's guideline export - their first line starts with "
+            "<!-- wikitool:export. They are generated from kb/ and never go back into raw/:\n"
+            f"{listed}\n  Remove them from incoming/ and accept again; nothing was moved."
+        )
+
+
+def _check_target(dst: Path, remedy: str) -> None:
+    problem = path_budget_problem_for(dst)
+    if problem:
+        raise _Refused(f"Cannot write {problem}. {remedy}")
+    if dst.exists():
+        raise _Refused(f"Cannot promote: {rel_path(dst)} already exists.")
+
+
+def _plan_file_moves(
+    resolved: list[Path], existing_raw_paths: list[Path], page: Optional[str]
+) -> list[tuple[Path, Path]]:
+    """Where each file goes - the bundle decision, the path budget, an existing
+    target and the name rule, all checked before anything moves. `page` and
+    `existing_raw_paths` are the `--page` case; `raw pending` asks with
+    neither."""
+    # A bundle directory forms once two or more files belong to the source
+    # (Gitea #58 decision 3): from the second file on, never before. Whenever
+    # --page targets an existing page it always has >=1 raw file already
+    # (types/source.md requires raw_files:), so bundling always applies there.
+    total = len(existing_raw_paths) + len(resolved)
+    bundle_dir: Optional[Path] = None
+    if len(existing_raw_paths) >= 2:
+        parents = {p.parent for p in existing_raw_paths}
+        if len(parents) != 1:
+            raise _Refused(
+                f"'{page}' raw_files: are not all in one directory - fix them by hand first "
+                "(see `sources coverage`)."
+            )
+        bundle_dir = parents.pop()
+    elif total >= 2:
+        if existing_raw_paths:
+            # Growing a bundle out of a single already-promoted file (Gitea
+            # #67 decision): the bundle forms at that file's own parent
+            # directory, never at today's shard - the file's capture date is
+            # whatever it always was, and a bundle mixing an old and a new
+            # shard would have no single correct address.
+            primary = existing_raw_paths[0]
+            bundle_dir = primary.parent / primary.stem
+        else:
+            primary = resolved[0]
+            bundle_dir = _shard_dir() / primary.stem
+
+    moves: list[tuple[Path, Path]] = []
+    for existing in existing_raw_paths:
+        if bundle_dir is not None and existing.parent != bundle_dir:
+            moves.append((existing, bundle_dir / existing.name))
+    for new_path in resolved:
+        dst = (bundle_dir / new_path.name) if bundle_dir is not None else (_shard_dir() / new_path.name)
+        moves.append((new_path, dst))
+
+    for _src, dst in moves:
+        _check_target(
+            dst,
+            "The name comes from the file in incoming/: rename it there to something shorter "
+            "and accept it again.",
+        )
+
+    # Stem uniqueness across raw/ (Gitea #64, widened by #67): the name this
+    # call is about to claim - the bundle's name, or the lone file's stem when
+    # no bundle forms - must not already belong to something this call does
+    # not itself own. "Owns" means: one of the page's already-registered raw
+    # files (the pitfall from the module docstring - a single file growing
+    # into a bundle of its own name momentarily still occupies that name), or,
+    # once a bundle already has >=2 registered files, the bundle directory
+    # itself.
+    claimed_name = bundle_dir.name if bundle_dir is not None else resolved[0].stem
+    occupied = _occupied_stems(config.RAW_DIR)
+    owned = set(existing_raw_paths)
+    if len(existing_raw_paths) >= 2:
+        owned.add(bundle_dir)
+    holder = occupied.get(claimed_name)
+    if holder is not None and holder not in owned:
+        raise _Refused(_stem_collision_message(claimed_name, holder))
+    return moves
+
+
+def _walk_folder(folder: Path) -> tuple[list[Path], list[Path], list[str]]:
+    """Every file below `folder`, every directory (`folder` included), and every
+    entry `raw accept` refuses to take - hidden or a symlink anywhere, or
+    neither a file nor a directory. Never descends into a refused entry, and
+    never follows a link."""
+    files: list[Path] = []
+    dirs: list[Path] = [folder]
+    refused: list[str] = []
+    for dirpath, dirnames, filenames in os.walk(folder, followlinks=False):
+        base = Path(dirpath)
+        descend = []
+        for name in sorted(dirnames + filenames):
+            entry = base / name
+            if name.startswith("."):
+                refused.append(f"{rel_path(entry)} (hidden)")
+            elif entry.is_symlink():
+                refused.append(f"{rel_path(entry)} (symlink)")
+            elif entry.is_dir():
+                dirs.append(entry)
+                descend.append(name)
+            elif entry.is_file():
+                files.append(entry)
+            else:
+                refused.append(f"{rel_path(entry)} (neither a file nor a directory)")
+        dirnames[:] = descend
+    return sorted(files), dirs, refused
+
+
+def _folder_collision_message(name: str, holder: Path) -> str:
+    if holder.is_dir() and (holder / repo_capture.MANIFEST_NAME).is_file():
+        return (
+            f'{rel_path(holder)} already claims the name "{name}" under raw/, and it is a captured '
+            "bundle.\n"
+            "  A new edition of it is captured and accepted as a whole:\n"
+            f"    tools/wikitool raw capture --update {rel_path(holder)}\n"
+            f"    tools/wikitool raw accept --replaces-bundle {rel_path(holder)} incoming/{name}\n"
+            "  A second, separate source gets a different --name."
+        )
+    return (
+        f'{rel_path(holder)} already claims the name "{name}" under raw/.\n'
+        "  Rename the folder in incoming/ (add a distinguishing suffix) and accept it again.\n"
+        "  A folder that is not a captured bundle has no --replaces-bundle: a later edition of "
+        "one file in it is replaced file by file."
+    )
+
+
+def _reserved_manifest_message(path: Path) -> str:
+    return (
+        f"{rel_path(path)} is named {repo_capture.MANIFEST_NAME}, which is reserved for the "
+        "manifest `raw capture` writes at the top of a captured bundle. Rename it in incoming/ and "
+        "accept it again."
+    )
+
+
+def _plan_folder(folder: Path) -> tuple[Path, list[tuple[Path, Path]], list[Path]]:
+    """The bundle directory for `raw accept incoming/<folder>`, every move into
+    it, and the directories to remove afterwards, deepest first - or
+    `_Refused`, before anything moves.
+
+    The invariant the cleanup rests on (Gitea #112): after the moves no file
+    is left below `folder`, so removing the emptied directories can never
+    take one with it. That holds only because every entry the move would
+    skip - hidden, a symlink, a special file - is refused here, up front."""
+    incoming = _incoming_dir()
+    try:
+        rel = folder.relative_to(incoming)
+    except ValueError:
+        raise _Refused(
+            f"{rel_path(folder)} is not under incoming/ - `raw accept` only takes a folder from "
+            "there. See raw/CONTRACT.md."
+        ) from None
+    if len(rel.parts) != 1:
+        raise _Refused(
+            f"incoming/{rel.as_posix()} is not directly in incoming/ - a folder is accepted only "
+            f"as a whole, from the top: raw accept incoming/{rel.parts[0]}"
+        )
+    if folder.name.startswith(".") or folder.is_symlink():
+        raise _Refused(f"{rel_path(folder)} is hidden or a symlink - raw accept does not take it.")
+
+    files, dirs, refused = _walk_folder(folder)
+    if refused:
+        listed = "\n".join(f"  - {entry}" for entry in refused)
+        raise _Refused(
+            f"{rel_path(folder)}/ holds entries raw accept does not take - hidden entries, "
+            f"symlinks and special files are refused, so nothing is left behind:\n{listed}\n"
+            "  Remove or replace them in incoming/, then accept the folder again."
+        )
+    if not files:
+        raise _Refused(f"{rel_path(folder)}/ holds no file - nothing to accept.")
+    stray = [f for f in files if f.name == repo_capture.MANIFEST_NAME and f.parent != folder]
+    if stray:
+        raise _Refused(_reserved_manifest_message(stray[0]))
+    _check_no_export(files)
+
+    holder = _occupied_stems(config.RAW_DIR).get(folder.name)
+    if holder is not None:
+        raise _Refused(_folder_collision_message(folder.name, holder))
+
+    bundle_dir = _shard_dir() / folder.name
+    moves = [(src, bundle_dir / src.relative_to(folder)) for src in files]
+    for _src, dst in moves:
+        _check_target(
+            dst,
+            "The path comes from the folder in incoming/: shorten the folder's name or the "
+            "names inside it, and accept it again.",
+        )
+    return bundle_dir, moves, sorted(dirs, key=lambda d: len(d.parts), reverse=True)
+
+
+def _capture_choices() -> tuple[list[str], list[str]]:
+    """Allowed `--fidelity`/`--authority` values, straight from the schema
+    (single source of truth) - `unknown` excluded, since it is backfill-only
+    and neither flag may write it (Gitea #67)."""
+    fidelity = [v for v in resolver.get_enum(_SOURCE_TYPE_PATH, "fidelity") if v != "unknown"]
+    authority = [v for v in resolver.get_enum(_SOURCE_TYPE_PATH, "authority") if v != "unknown"]
+    return fidelity, authority
+
+
+def _check_capture_value(field: str, value: Optional[str], allowed: list[str]) -> None:
+    if value is None:
+        return
+    if value == "unknown" or value not in allowed:
+        fail(
+            f"--{field} must be one of: {', '.join(allowed)}. 'unknown' is backfill-only - only "
+            "`wikitool touch` may write it, on a page predating this rule."
+        )
+
+
+def _check_matches_manifest(folder: Path, manifest: repo_capture.Manifest) -> None:
+    """A captured folder holds exactly the files its manifest lists, plus the
+    manifest - anything else was added or removed by hand after `raw capture`,
+    and the manifest would then describe a bundle that does not exist."""
+    on_disk = set(repo_capture.bundle_files(folder))
+    listed = set(manifest.files)
+    if on_disk != listed:
+        extra = sorted(on_disk - listed)
+        missing = sorted(listed - on_disk)
+        lines = [f"  + {p} (not in the manifest)" for p in extra]
+        lines += [f"  - {p} (listed, but missing)" for p in missing]
+        raise _Refused(
+            f"{rel_path(folder)}/ no longer matches its {repo_capture.MANIFEST_NAME}:\n"
+            + "\n".join(lines)
+            + "\n  A captured bundle is not edited in incoming/. Remove the folder and capture it "
+            "again."
+        )
+
+
+def _read_manifest_or_refuse(path: Path) -> repo_capture.Manifest:
+    try:
+        return repo_capture.read_manifest(path)
+    except ValidationError as exc:
+        raise _Refused(f"{rel_path(path)}: {exc}") from None
+
+
+def _check_manifest_capture_values(manifest: repo_capture.Manifest, path: Path) -> None:
+    fidelity_choices, authority_choices = _capture_choices()
+    for name, value, allowed in (
+        ("fidelity", manifest.fidelity, fidelity_choices),
+        ("authority", manifest.authority, authority_choices),
+    ):
+        if value not in allowed:
+            raise _Refused(
+                f"{rel_path(path)} names {name}={value!r}, which is not one of: {', '.join(allowed)}. "
+                "Capture it again with a valid value."
+            )
+
+
+def _plan_captured_folder(folder: Path):
+    plan = _plan_folder(folder)
+    manifest_path = folder / repo_capture.MANIFEST_NAME
+    manifest = _read_manifest_or_refuse(manifest_path)
+    _check_manifest_capture_values(manifest, manifest_path)
+    _check_matches_manifest(folder, manifest)
+    return plan, manifest
+
+
+def _change_groups(
+    changes: list[tuple[str, str]], pages
+) -> tuple[list[tuple[Optional[str], list[tuple[str, str]]]], list[str]]:
+    """`changes` (`(status, raw path)`) grouped by owning source page - a new
+    file has none yet, so it lands in the `None` group - and every owning page
+    of the bundle's existing files. A footnote's file qualifier is a free
+    string, so citing pages can only be reported per source page, not per
+    file."""
+    by_raw = source_pages_by_raw_file(pages)
+    groups: dict[Optional[str], list[tuple[str, str]]] = {}
+    for status, path in changes:
+        owners = sorted(set(by_raw.get(path, []))) if status != "A" else []
+        groups.setdefault(owners[0] if owners else None, []).append((status, path))
+    ordered = sorted(groups.items(), key=lambda item: (item[0] is None, item[0] or ""))
+    return ordered, sorted({t for _s, p in changes for t in by_raw.get(p, [])})
+
+
+def _render_changes(changes: list[tuple[str, str]], pages, indent: str) -> list[str]:
+    groups, _owners = _change_groups(changes, pages)
+    lines: list[str] = []
+    for title, entries in groups:
+        if title is None:
+            lines.append(f"{indent}not covered by any source page:")
+        else:
+            lines.append(f"{indent}[[{title}]]")
+        lines.extend(f"{indent}  {status} {path}" for status, path in entries)
+        if title is not None:
+            citers = citing_pages(pages, title)
+            lines.append(
+                f"{indent}  cited by: " + (", ".join(f"[[{c}]]" for c in citers) if citers else "(nothing yet)")
+            )
+    return lines
+
+
+def _write_exclusive(writes: list[tuple[Path, bytes]]) -> None:
+    """Create every file or none: `x` mode refuses a file that appeared since
+    the check above, and a failure part-way removes what this call wrote."""
+    written: list[Path] = []
+    made: list[Path] = []
+    try:
+        for path, data in writes:
+            missing = [d for d in reversed(path.parents) if not d.exists()]
+            path.parent.mkdir(parents=True, exist_ok=True)
+            made.extend(missing)
+            with path.open("xb") as handle:
+                written.append(path)
+                handle.write(data)
+    except OSError as exc:
+        for path in written:
+            path.unlink(missing_ok=True)
+        for directory in reversed(made):
+            try:
+                directory.rmdir()
+            except OSError:
+                pass
+        fail(escape(f"Could not write {rel_path(path)}: {exc}. Nothing was left in incoming/."))
+
+
+def _derive_hint(path: Path) -> Optional[str]:
+    """The `raw derive` line for a file a filter reads, else None."""
+    if filters.filter_for(path.name) is None:
+        return None
+    return f"tools/wikitool raw derive {rel_path(path)}"
+
+
+def _unusable_name(name: str) -> bool:
+    """Not one plain directory name: empty, padded, hidden, a path, a control
+    character, or the manifest's reserved name."""
+    return (
+        not name.strip()
+        or name != name.strip()
+        or name.startswith(".")
+        or name == repo_capture.MANIFEST_NAME
+        or any(c in name for c in "/\\:")
+        or any(ord(c) < 0x20 for c in name)
+    )

tools/chemenu/commands/raw/accept.py

@@ -0,0 +1,948 @@
+"""`wikitool raw accept` - promote one or more files from `incoming/` into
+`raw/`, with the destination computed rather than chosen by hand (Gitea #58),
+sharded by the accept date rather than by a hand-picked type (Gitea #67).
+
+A human no longer classifies a file at all - `raw/` is an address computed
+purely from *when* the file was accepted, and the kind of source comes from
+its content (`source_type:` on the source page). So a subdirectory of
+`incoming/` carries no type any more, and since Gitea #112 it is not tolerated
+as one either: it **is** a source, accepted as a whole. A file argument must
+sit directly in `incoming/`:
+
+- **Single file, no bundle.** One file promoted alone lands as
+  `raw/<YYYY>/<MM>/<name>` - no directory of its own.
+- **Bundle from the second file on.** Several files of one source promoted in
+  the same call land under `raw/<YYYY>/<MM>/<stem>/`, named after the first
+  file's stem.
+- **A folder is one source** (Gitea #112). `raw accept incoming/<folder>`
+  moves every file below it to `raw/<YYYY>/<MM>/<folder>/` at the same
+  relative path, then removes the directories that are left empty. The folder
+  name is the bundle name.
+- **Growing an existing single file into a bundle.** `--page` extends an
+  existing source page's `raw_files:`. If that raises the page from one file
+  to more than one, the file it already had is folded into a bundle at its
+  own parent directory - `raw/<its-existing-location>/<stem>/` - not at
+  today's shard, so a bundle never mixes an old capture date with today's
+  (Gitea #67 decision, "Datums-Shard" § "Bündelort").
+
+Existing files under `raw/` are never moved by this change (Gitea #67
+"Altbestand bleibt stehen"): `raw/articles/`, `raw/documents/`, `raw/notes/`
+and `raw/assets/` keep whatever they already held, and stay valid promotion
+targets for `--replaces`.
+
+**Every promotion now also carries `--fidelity` and `--authority`** (Gitea
+#67): how faithfully the material was captured, and what it is entitled to
+claim about its subject. Neither has a default and neither may be `unknown`
+here - that value is backfill-only, written only by `wikitool touch` on a
+page predating this rule. Given `--page`, both are written straight onto the
+target page (once - fill-once, like `touch`, see `touch._capture_field_or_fail`);
+without `--page` there is no page yet to write them onto (`wiki-ingest`
+creates it afterwards), so this command instead prints the exact
+`wikitool new source --set fidelity=... --set authority=...` follow-up line,
+and `new source` itself refuses to scaffold a source page without both.
+`--replaces` is the one path where both become *optional*: passing them there
+is the sanctioned way to correct an already-set capture value on a later
+edition, the same way `touch`'s own refusal points back to `--replaces`.
+
+Multi-owner raw files (`provenance.duplicate_raw_file_owners`) are refused
+rather than silently moved: relocating a file another page also claims would
+break that page's `raw_files:` without it ever being consulted.
+
+**Stem uniqueness across the whole of `raw/`** (Gitea #64, widened by #67):
+without it, a second, unrelated source whose primary file happens not to
+collide on the exact filename slips silently into an existing bundle, because
+the per-file `dst.exists()` check above never looks at the bundle directory
+itself. The set of names occupied anywhere under `raw/` - file stems and
+bundle directory names alike, at whichever level directly holds files - must
+stay unique; `_occupied_stems()` and the check built on it enforce that,
+while still allowing a call to grow a bundle it already owns (via `--page`,
+or by continuing an existing bundle). It used to be scoped to one type
+directory; #67 removes type directories from the addressing scheme entirely,
+so uniqueness now has to span old-style flat directories and the new date
+shard together, or `--replaces` on a bygone stem would be ambiguous across
+shards.
+
+**`--replaces <raw-path>`** is the only sanctioned way past that rule: whether
+a new file is a later edition of an existing source or a second, separate one
+is a human's decision, never the tool's or an agent's, so the command refuses
+and names both routes rather than choosing one (Gitea #64 decision 2).
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+from rich.markup import escape
+
+from chemenu import cli_contract, config, repo_capture
+from chemenu.commands._util import fail, path_budget_problem_for, rel_path, success
+from chemenu.frontmatter_io import write_page
+from chemenu.kb_scan import load_kb_pages
+from chemenu.provenance import citing_pages, source_pages_by_raw_file, source_raw_files
+from chemenu.commands.raw._common import (
+    _Refused,
+    _capture_choices,
+    _check_capture_value,
+    _check_files,
+    _check_manifest_capture_values,
+    _check_matches_manifest,
+    _check_no_export,
+    _derive_hint,
+    _incoming_dir,
+    _plan_captured_folder,
+    _plan_file_moves,
+    _plan_folder,
+    _read_manifest_or_refuse,
+    _refusals_fail,
+    _render_changes,
+    _resolve,
+    _validate_under_incoming,
+    _walk_folder,
+)
+
+
+def _captured_bundle_message(path: Path, bundle: Path, flag: str) -> str:
+    return (
+        f"{rel_path(path)} lies in the captured bundle {rel_path(bundle)}/ - {flag} would detach the "
+        "bundle's content from the commit its manifest names. A captured bundle changes only as a "
+        "whole:\n"
+        f"  tools/wikitool raw capture --update {rel_path(bundle)}\n"
+        f"  tools/wikitool raw accept --replaces-bundle {rel_path(bundle)} incoming/{bundle.name}"
+    )
+
+
+def _overwrite_capture_fields_on_page(page, fidelity: Optional[str], authority: Optional[str]) -> bool:
+    """Write `fidelity`/`authority` onto `page.frontmatter`, unconditionally -
+    only called from `_replace`, the one sanctioned way to correct an
+    already-set capture value (Gitea #67). Returns whether anything changed,
+    so the caller only writes the page back when it needs to."""
+    changed = False
+    for field, value in (("fidelity", fidelity), ("authority", authority)):
+        if value is not None and page.frontmatter.get(field) != value:
+            page.frontmatter[field] = value
+            changed = True
+    return changed
+
+
+def _replace(
+    files: list[Path],
+    replaces: Path,
+    page: Optional[str],
+    fidelity: Optional[str],
+    authority: Optional[str],
+    dry_run: bool,
+) -> None:
+    """`raw accept --replaces <target> <incoming file>` - overwrite an
+    existing raw/ file wholesale with a later edition, in place, with
+    `raw_files:` on every source page left untouched (Gitea #64 decision 2).
+    `--fidelity`/`--authority` are optional here, and are the one sanctioned
+    way to correct an already-set capture value (Gitea #67) - passed, they
+    overwrite the owning page's value; omitted, the page's capture fields are
+    left exactly as they were.
+
+    Every check below runs before any write, so a failure leaves both the
+    target and the incoming file exactly as they were - the invariant the
+    issue names for this destructive step.
+    """
+    if page is not None:
+        fail("--replaces and --page cannot be combined - a replacement never changes raw_files:.")
+    if len(files) != 1:
+        fail("--replaces takes exactly one incoming file - a replacement is one file for one file.")
+
+    fidelity_choices, authority_choices = _capture_choices()
+    _check_capture_value("fidelity", fidelity, fidelity_choices)
+    _check_capture_value("authority", authority, authority_choices)
+
+    incoming_path = _resolve(files[0])
+    if not incoming_path.is_file():
+        fail(f"{rel_path(incoming_path)} does not exist or is not a file.")
+    _validate_under_incoming(incoming_path, _incoming_dir())
+    _refusals_fail(_check_no_export, [incoming_path])
+
+    target = _resolve(replaces)
+    try:
+        target.relative_to(config.RAW_DIR)
+    except ValueError:
+        fail(f"--replaces target {rel_path(target)} does not lie under raw/.")
+    if not target.is_file():
+        fail(f"--replaces target {rel_path(target)} does not exist or is not a file.")
+    captured = repo_capture.captured_bundle_of(target, config.RAW_DIR)
+    if captured is not None:
+        fail(escape(_captured_bundle_message(target, captured, "--replaces")))
+    if target.name != incoming_path.name:
+        fail(
+            f"--replaces target {rel_path(target)} has a different filename than "
+            f"{rel_path(incoming_path)} - a rename is not part of a replacement (see `raw rename`, #16)."
+        )
+
+    pages = load_kb_pages(config.KB_DIR)
+    by_raw = source_pages_by_raw_file(pages)
+    owners = sorted(set(by_raw.get(rel_path(target), [])))
+    if len(owners) > 1:
+        fail(
+            f"Cannot replace {rel_path(target)}: more than one source page claims it "
+            f"({', '.join(owners)}). Resolve the multiple ownership first (see `wikitool sources coverage`)."
+        )
+
+    owner_page = pages.get(owners[0]) if owners else None
+    will_update_capture = owner_page is not None and _overwrite_capture_fields_on_page(
+        owner_page, fidelity, authority
+    )
+
+    if dry_run:
+        typer.echo(f"[dry-run] would replace {rel_path(target)} with {rel_path(incoming_path)}. No files written.")
+        if will_update_capture:
+            typer.echo(
+                f"[dry-run] would set fidelity={fidelity!r}, authority={authority!r} on '{owners[0]}'"
+            )
+        return
+
+    target.unlink()
+    incoming_path.rename(target)
+
+    if will_update_capture:
+        write_page(owner_page.path, owner_page.frontmatter, owner_page.body)
+
+    if not owners:
+        success(
+            f"Replaced {rel_path(target)} (previous version stays in git history). "
+            "No source page covers this file - see `wikitool sources coverage`."
+        )
+        return
+
+    source_title = owners[0]
+    citers = citing_pages(pages, source_title)
+    typer.echo(f"Replaced {rel_path(target)} (previous version stays in git history).")
+    typer.echo("These pages were compiled against the previous version and may now be stale:")
+    typer.echo(f"  [[{source_title}]]")
+    if citers:
+        for citer in citers:
+            typer.echo(f"    cited by: [[{citer}]]")
+    else:
+        typer.echo("    cited by: (nothing yet)")
+    success("Review them in this same run: the replacement and their update belong in one commit.")
+
+
+def _accept_captured_folder(
+    folder: Path, fidelity: Optional[str], authority: Optional[str], dry_run: bool
+) -> None:
+    """`raw accept incoming/<bundle>` for a folder `raw capture` wrote: the
+    capture fields come from its manifest and nowhere else - a second source
+    for the same value is the second declaration the manifest exists to rule
+    out. `raw capture --update` is how they are corrected."""
+    if fidelity is not None or authority is not None:
+        fail(escape(
+            f"{rel_path(folder)}/ is a captured bundle: --fidelity and --authority come from its "
+            f"{repo_capture.MANIFEST_NAME}, not from the command line. Accept it without them; to "
+            "change a value, capture it again with raw capture --fidelity/--authority."
+        ))
+    (_bundle, _moves, _dirs), manifest = _refusals_fail(_plan_captured_folder, folder)
+    _accept_folder(folder, manifest.fidelity, manifest.authority, dry_run, manifest=manifest)
+
+
+def _accept_folder(
+    folder: Path,
+    fidelity: str,
+    authority: str,
+    dry_run: bool,
+    manifest: Optional[repo_capture.Manifest] = None,
+) -> None:
+    """`raw accept incoming/<folder>` - one folder, one source (Gitea #112).
+
+    Not atomic: one move per file, then the emptied directories. A failure
+    part-way leaves a half-accepted folder, which is reported, not resumed -
+    the call is not repeated (tool error contract, case 4)."""
+    bundle_dir, moves, dirs = _refusals_fail(_plan_folder, folder)
+
+    if dry_run:
+        for src, dst in moves:
+            typer.echo(f"[dry-run] would move {rel_path(src)} -> {rel_path(dst)}")
+        typer.echo(f"[dry-run] would remove {rel_path(folder)}/ once it is empty")
+        typer.echo(f"[dry-run] would move {len(moves)} file(s). No files written.")
+        return
+
+    for src, dst in moves:
+        dst.parent.mkdir(parents=True, exist_ok=True)
+        src.rename(dst)
+        typer.echo(f"  moved {rel_path(src)} -> {rel_path(dst)}")
+    # rmdir, never a recursive delete: it refuses a directory that still holds
+    # anything, so this step can only ever remove what the moves emptied.
+    left = []
+    for directory in dirs:
+        try:
+            directory.rmdir()
+        except OSError:
+            left.append(directory)
+    if left:
+        typer.echo(
+            f"WARN {', '.join(rel_path(d) for d in left)} could not be removed - something appeared "
+            "in it during the move. Every file of the folder was moved; look at what is left."
+        )
+
+    sources = [dst for _src, dst in moves if dst != bundle_dir / repo_capture.MANIFEST_NAME]
+    raw_files_arg = ",".join(rel_path(dst) for dst in sources)
+    captured_note = ""
+    if manifest is not None:
+        captured_note = (
+            f"  Captured from {manifest.repo} at {manifest.commit[:12]}; {repo_capture.MANIFEST_NAME} "
+            "is the bundle's manifest, not a source - it goes in no raw_files:.\n"
+            "  Cite a file of it with --file <path inside the bundle> (docs/runbook.md), not its "
+            "base name.\n"
+        )
+    success(
+        f"Promoted {rel_path(folder)}/ ({len(sources)} file(s)) to {rel_path(bundle_dir)}/.\n"
+        + captured_note
+        + "  Next:\n"
+        "  tools/wikitool new source --name \"<Title>\" \\\n"
+        f"    --set raw_files={raw_files_arg} \\\n"
+        f"    --set fidelity={fidelity} --set authority={authority} \\\n"
+        "    --set source_type=<category>\n"
+        "  Past the thresholds in instructions/ingest-large-tree.md, continue there instead:\n"
+        f"  tools/wikitool work new --input {rel_path(bundle_dir)}"
+    )
+
+
+# --- raw accept --replaces-bundle --------------------------------------------
+#
+# A captured bundle is replaced as a whole, at the address it already has: the
+# new edition comes from `raw capture --update`, which writes the same bundle
+# name into `incoming/`. Like `--replaces`, `raw_files:` stays as it is - which
+# source page takes a new file is an ingest judgment, not mechanics.
+
+@dataclass
+class _BundlePlan:
+    folder: Path
+    bundle: Path
+    manifest: repo_capture.Manifest
+    old_manifest: repo_capture.Manifest
+    changes: list[tuple[str, str]]  # (status, repo path)
+    unchanged: list[str]
+    incoming_dirs: list[Path]
+    owners: list[str]
+
+
+def _plan_bundle_replacement(folder: Path, bundle: Path, pages) -> _BundlePlan:
+    """Every check `--replaces-bundle` makes, all before anything moves - a
+    refusal leaves `raw/` and `incoming/` exactly as they were."""
+    incoming = _incoming_dir()
+    try:
+        rel = folder.relative_to(incoming)
+    except ValueError:
+        raise _Refused(f"{rel_path(folder)} is not under incoming/.") from None
+    if len(rel.parts) != 1 or not folder.is_dir() or folder.is_symlink():
+        raise _Refused(f"{rel_path(folder)} is not a folder directly in incoming/.")
+    try:
+        bundle.relative_to(config.RAW_DIR)
+    except ValueError:
+        raise _Refused(f"--replaces-bundle {rel_path(bundle)} does not lie under raw/.") from None
+    if not bundle.is_dir():
+        raise _Refused(f"--replaces-bundle {rel_path(bundle)} is not a directory under raw/.")
+    for side in (bundle, folder):
+        if not (side / repo_capture.MANIFEST_NAME).is_file():
+            raise _Refused(
+                f"{rel_path(side)}/ has no {repo_capture.MANIFEST_NAME} - only a bundle `raw capture` "
+                "wrote is replaced as a whole. A file of any other bundle is replaced file by file "
+                "with --replaces."
+            )
+    old_manifest = _read_manifest_or_refuse(bundle / repo_capture.MANIFEST_NAME)
+    manifest = _read_manifest_or_refuse(folder / repo_capture.MANIFEST_NAME)
+    if old_manifest.repo != manifest.repo:
+        raise _Refused(
+            f"{rel_path(folder)}/ was captured from {manifest.repo}, but {rel_path(bundle)}/ from "
+            f"{old_manifest.repo} - a bundle is only replaced by a new edition of the same repository."
+        )
+    if folder.name != bundle.name:
+        raise _Refused(
+            f"incoming/{folder.name} has a different name than {rel_path(bundle)} - a rename is not "
+            "part of a replacement. raw capture --update writes the bundle's own name."
+        )
+    _check_manifest_capture_values(manifest, folder / repo_capture.MANIFEST_NAME)
+
+    _files, incoming_dirs, refused = _walk_folder(folder)
+    _files_old, _dirs_old, refused_old = _walk_folder(bundle)
+    if refused or refused_old:
+        listed = "\n".join(f"  - {entry}" for entry in refused + refused_old)
+        raise _Refused(
+            f"Hidden entries, symlinks and special files are not replaced - nothing would be left "
+            f"exactly as the manifest says:\n{listed}"
+        )
+    _check_matches_manifest(folder, manifest)
+
+    new_files = repo_capture.bundle_files(folder)
+    _check_no_export(list(new_files.values()))
+    old_files = repo_capture.bundle_files(bundle)
+    changes = repo_capture.diff(old_files, {r: p.read_bytes() for r, p in new_files.items()})
+    changed = {r for _s, r in changes}
+    unchanged = sorted(r for r in new_files if r not in changed)
+
+    problems = [
+        problem
+        for status, r in changes
+        if status != "D" and (problem := path_budget_problem_for(bundle / r))
+    ]
+    if problems:
+        listed = "\n".join(f"  - {p}" for p in problems)
+        raise _Refused(f"Cannot write:\n{listed}\n  Narrow the capture's --path globs.")
+
+    by_raw = source_pages_by_raw_file(pages)
+    multi = [
+        (rel_path(path), sorted(set(by_raw.get(rel_path(path), []))))
+        for path in old_files.values()
+        if len(set(by_raw.get(rel_path(path), []))) > 1
+    ]
+    if multi:
+        listed = "\n".join(f"  - {p}: {', '.join(o)}" for p, o in multi)
+        raise _Refused(
+            "Cannot replace: more than one source page claims these files of the bundle:\n"
+            f"{listed}\n  Resolve the multiple ownership first (see `wikitool sources coverage`)."
+        )
+    owners = sorted({t for path in old_files.values() for t in by_raw.get(rel_path(path), [])})
+    return _BundlePlan(
+        folder, bundle, manifest, old_manifest, changes, unchanged,
+        sorted(incoming_dirs, key=lambda d: len(d.parts), reverse=True), owners,
+    )
+
+
+def _remove_empty_dirs(top: Path) -> None:
+    """`rmdir` every directory below `top` that is empty, deepest first -
+    never recursive, so nothing that still holds a file can go."""
+    for dirpath, _dirnames, _filenames in os.walk(top, topdown=False):
+        if Path(dirpath) != top:
+            try:
+                Path(dirpath).rmdir()
+            except OSError:
+                pass
+
+
+def _replace_bundle(
+    files: list[Path],
+    replaces_bundle: Path,
+    page: Optional[str],
+    replaces: Optional[Path],
+    fidelity: Optional[str],
+    authority: Optional[str],
+    dry_run: bool,
+) -> None:
+    """`raw accept --replaces-bundle <raw-bundle> incoming/<bundle>`.
+
+    The invariant this destructive step keeps: afterwards the bundle under
+    `raw/` holds exactly the files of the new manifest plus `_capture.json`,
+    at the address it already had, and `incoming/<bundle>/` is gone. Not
+    atomic - deletes, then moves, then the manifest - and a failure part-way
+    is reported, not resumed."""
+    if page is not None or replaces is not None:
+        fail("--replaces-bundle cannot be combined with --page or --replaces - a bundle replacement "
+             "never changes raw_files: and takes the whole bundle at once.")
+    if len(files) != 1:
+        fail("--replaces-bundle takes exactly one folder in incoming/ - the new edition of the bundle.")
+    if fidelity is not None or authority is not None:
+        fail(f"--fidelity and --authority come from the new edition's {repo_capture.MANIFEST_NAME}. "
+             "Accept it without them; to change a value, capture it again with "
+             "raw capture --update --fidelity/--authority.")
+
+    pages = load_kb_pages(config.KB_DIR)
+    plan = _refusals_fail(_plan_bundle_replacement, _resolve(files[0]), _resolve(replaces_bundle), pages)
+    bundle, folder = plan.bundle, plan.folder
+    raw_changes = [(status, rel_path(bundle / r)) for status, r in plan.changes]
+    owner_pages = [pages[t] for t in plan.owners if t in pages]
+    capture_updates = [
+        p for p in owner_pages
+        if _overwrite_capture_fields_on_page(p, plan.manifest.fidelity, plan.manifest.authority)
+    ]
+
+    head = (
+        f"{rel_path(bundle)}/: {plan.old_manifest.commit[:12]} -> {plan.manifest.commit[:12]} - "
+        f"{sum(s == 'A' for s, _ in plan.changes)} added, {sum(s == 'M' for s, _ in plan.changes)} "
+        f"modified, {sum(s == 'D' for s, _ in plan.changes)} removed, {len(plan.unchanged)} unchanged."
+    )
+    if dry_run:
+        typer.echo(f"[dry-run] would replace {head}")
+        for line in _render_changes(raw_changes, pages, "  "):
+            typer.echo(line)
+        for p in capture_updates:
+            typer.echo(f"[dry-run] would set fidelity={plan.manifest.fidelity!r}, "
+                       f"authority={plan.manifest.authority!r} on '{p.title}'")
+        typer.echo("[dry-run] No files written.")
+        return
+
+    for status, r in plan.changes:
+        if status == "D":
+            (bundle / r).unlink()
+    _remove_empty_dirs(bundle)
+    for status, r in plan.changes:
+        if status in ("A", "M"):
+            dst = bundle / r
+            dst.parent.mkdir(parents=True, exist_ok=True)
+            os.replace(folder / r, dst)
+    for r in plan.unchanged:
+        (folder / r).unlink()
+    os.replace(folder / repo_capture.MANIFEST_NAME, bundle / repo_capture.MANIFEST_NAME)
+    left = []
+    for directory in plan.incoming_dirs:
+        try:
+            directory.rmdir()
+        except OSError:
+            left.append(directory)
+    if left:
+        typer.echo(
+            f"WARN {', '.join(rel_path(d) for d in left)} could not be removed - something appeared "
+            "in it during the replacement. Look at what is left."
+        )
+    for p in capture_updates:
+        write_page(p.path, p.frontmatter, p.body)
+        typer.echo(f"  set fidelity={plan.manifest.fidelity}, authority={plan.manifest.authority} "
+                   f"on '{p.title}'")
+
+    typer.echo(f"Replaced {head} The previous edition stays in git history.")
+    if not raw_changes:
+        success("Nothing inside the globs changed; only the manifest moved to the new commit.")
+        return
+    for line in _render_changes(raw_changes, pages, "  "):
+        typer.echo(line)
+
+    by_raw = source_pages_by_raw_file(pages)
+    sole_owner = plan.owners[0] if len(plan.owners) == 1 else None
+    next_lines: list[str] = []
+    removed: dict[str, list[str]] = {}
+    for status, path in raw_changes:
+        if status == "D":
+            for title in sorted(set(by_raw.get(path, []))):
+                removed.setdefault(title, []).append(path)
+                next_lines.append(f'  tools/wikitool touch --page "{title}" --remove raw_files={path}')
+        elif status == "A":
+            target = sole_owner or "<Source page>"
+            next_lines.append(
+                f'  tools/wikitool touch --page "{target}" --add raw_files={path}   (or a new source page)'
+            )
+    for title, paths in sorted(removed.items()):
+        page_obj = pages.get(title)
+        if page_obj is not None and set(source_raw_files(page_obj)) <= set(paths):
+            next_lines.append(
+                f"  '{title}' is left without raw_files: - instructions/page-lifecycle.md § Delete"
+            )
+    if next_lines:
+        typer.echo("  Next:")
+        for line in next_lines:
+            typer.echo(line)
+    success("Review the pages above in this same run: the replacement and their update belong in one "
+            "commit. `git diff` on the bundle shows the `M` and `D` files; the `A` files are only in "
+            "the list above - they are still untracked.")
+
+
+def _warn_underived(resolved: list[Path]) -> None:
+    """A file `raw derive` could read, promoted with no `.md` beside it, leaves
+    the source without a text the wiki can read - say so, and carry on."""
+    stems_with_text = {p.stem for p in resolved if p.suffix.lower() == ".md"}
+    for path in resolved:
+        hint = _derive_hint(path)
+        if hint is not None and path.stem not in stems_with_text:
+            typer.echo(
+                f"WARN {rel_path(path)} has a text filter and no derived .md beside it - the "
+                f"source would have no text to read. Run {hint} first and accept the folder it "
+                "makes, unless the file is meant to be kept as it is."
+            )
+
+
+@cli_contract.record(cli_contract.CommandRecord(
+    path="raw accept",
+    summary="Promote one or more files, or one folder, from `incoming/` into `raw/`.",
+    synopsis=(
+        cli_contract.Variant(
+            usage='raw accept <file> [<file> ...] --fidelity <v> --authority <v> '
+            '[--page "<Title>"] [--dry-run]',
+            notes="Promote one or more files from `incoming/` into today's `raw/<YYYY>/<MM>/` "
+            "shard",
+        ),
+        cli_contract.Variant(
+            usage="raw accept incoming/<folder> --fidelity <v> --authority <v> [--dry-run]",
+            notes="Promote a whole folder as one source, its structure kept, into "
+            "`raw/<YYYY>/<MM>/<folder>/`",
+        ),
+        cli_contract.Variant(
+            usage='raw accept <file> --replaces <raw-path> [--fidelity <v>] [--authority <v>] '
+            "[--dry-run]",
+            notes="Overwrite one existing raw file in place with a new edition",
+        ),
+        cli_contract.Variant(
+            usage="raw accept incoming/<bundle> --replaces-bundle <raw-bundle> [--dry-run]",
+            notes="Replace a captured bundle as a whole with the new edition `raw capture "
+            "--update` wrote",
+        ),
+    ),
+    properties=cli_contract.Properties(
+        effect=cli_contract.Effect.WRITE,
+        idempotent=cli_contract.Idempotent.NO,
+        atomic="`raw accept`: No - one filesystem move per file, then (with `--page`) one page "
+        "write. With a folder: No - one move per file, then one `rmdir` per emptied directory; "
+        "a half-accepted folder is not resumed. `raw accept --replaces`: No - one `unlink()` + "
+        "one `rename()`, plus (if `--fidelity`/`--authority` was given) one page write. "
+        "`raw accept --replaces-bundle`: No - one `unlink()` per removed file, one `rename()` per "
+        "added or modified file, then the manifest, then one `rmdir` per emptied directory, plus "
+        "one page write per owning page whose capture fields change; a half-replaced bundle is "
+        "not resumed",
+        budget=cli_contract.Budget.COUNTED,
+    ),
+    notes=(
+        "Promotes files from `incoming/` into `raw/<YYYY>/<MM>/`, computed from the accept "
+        "date rather than chosen by hand. A file argument must sit directly in `incoming/`; a "
+        "file inside a subdirectory is refused, since a subdirectory is a source of its own.",
+        "A folder argument (`incoming/<folder>`) is one source: every file below it moves to "
+        "`raw/<YYYY>/<MM>/<folder>/` at the same relative path, and the directories left empty "
+        "are removed - `incoming/<folder>` no longer exists afterwards. The folder name is the "
+        "bundle name; two `README.md` in different subfolders are no conflict.",
+        "A folder is accepted alone - no other argument, no `--page`, no `--replaces` - with "
+        "one `--fidelity`/`--authority` pair for all of it. It is refused, before anything "
+        "moves, if it is empty, or if a hidden entry (name starting with `.`), a symlink or a "
+        "special file sits anywhere below it; the refusal names each one.",
+        "One file promoted alone lands with no directory of its own; several files in one call "
+        "nest under `raw/<YYYY>/<MM>/<stem>/`, named after the first file's stem.",
+        "`--fidelity`/`--authority` are required on a plain accept (`types describe source` "
+        "lists the values); `unknown` is refused - it is backfill-only.",
+        "`--page \"<Title>\"` additionally extends that existing source page's `raw_files:` in "
+        "the same call and writes both capture fields onto it - refused if it already carries a "
+        "different value, since a capture field is fixed once.",
+        "If `--page` raises the page past one file, its already-promoted file is folded into a "
+        "bundle at *its own* parent directory, not today's shard, so a bundle never mixes an "
+        "old and a new capture date - after checking that file has no other owner.",
+        "Every name occupied anywhere under `raw/` - file stems and bundle directory names "
+        "alike, old type directories and date shards together - stays unique: a promote whose "
+        "target name already belongs to something this call does not itself own is refused, "
+        "naming `--replaces` and renaming in `incoming/` as the two routes, without "
+        "recommending either.",
+        "`--replaces <raw-path>` overwrites that file in place with the single incoming file "
+        "(same filename required), leaves every page's `raw_files:` untouched and writes no "
+        "`kb/` page; the previous edition survives only in `git log --follow <raw-path>`.",
+        "With `--replaces`, `--fidelity`/`--authority` are optional, and passing one overwrites "
+        "the owning page's already-set value - the one path the fixed-once rule does not "
+        "block.",
+        "`--replaces` refuses a target with more than one owning source page; with none, it "
+        "replaces anyway and says so. It cannot be combined with `--page` or with more than one "
+        "incoming file.",
+        "`--replaces` prints the source page (if any) and its citing pages, so their update "
+        "lands in the same commit as the replacement.",
+        "A file already at its computed destination is what \"already exists\" reports, not a "
+        "partial prior run to resume - safe to retry as-is once a cause is fixed.",
+        "A folder carrying `_capture.json` at its top is a captured bundle (`raw capture`): "
+        "`--fidelity`/`--authority` come from that manifest only and are refused on the command "
+        "line; the folder must hold exactly the files the manifest lists. `_capture.json` goes in "
+        "no `raw_files:`.",
+        "`--replaces-bundle <raw-bundle>` replaces a captured bundle under `raw/` with the new "
+        "edition in `incoming/<bundle>` at the same address: files gone from the new edition are "
+        "removed, emptied directories `rmdir`ed, and afterwards the bundle holds exactly the new "
+        "manifest's files plus `_capture.json`. `raw_files:` is left untouched; an owning source "
+        "page's `fidelity`/`authority` is overwritten where the new manifest differs.",
+        "`--replaces-bundle` prints each changed file as `A`/`M`/`D`, grouped by owning source page "
+        "with its citing pages, and the `touch --page ... --add/--remove raw_files=` lines that "
+        "follow; `git diff` on the bundle shows the `M` and `D` files, the `A` files appear only in "
+        "that list because they are still untracked.",
+        "`--replaces` and `--page` refuse a target inside a captured bundle; the refusal names "
+        "`raw capture --update` and `--replaces-bundle`. A file named `_capture.json` anywhere but "
+        "at a captured folder's top is refused - the name is reserved.",
+        "`--dry-run` reports the moves without making them.",
+    ),
+    failures=(
+        cli_contract.Failure(
+            label="raw accept",
+            cause="A file does not exist or is not directly in `incoming/`; two files in one "
+            "call share a filename; a target path already exists; or a target path would be "
+            "over the path budget (160 UTF-16 code units below the instance root)",
+            reaction="Fix the named argument and retry once. A file inside a subdirectory of "
+            "`incoming/` is accepted with its whole folder (`raw accept incoming/<folder>`) or "
+            "moved up into `incoming/` first. For a path over the budget, rename the file in "
+            "`incoming/` to something shorter - the refusal comes before anything moves, so "
+            "`incoming/` and `raw/` are unchanged",
+        ),
+        cli_contract.Failure(
+            label="raw accept incoming/<folder>",
+            cause="The folder is not directly in `incoming/`, is combined with another argument, "
+            "`--page` or `--replaces`, holds no file, or holds a hidden entry, a symlink or a "
+            "special file; a target path is over the budget; or the folder name is already "
+            "occupied under `raw/`",
+            reaction="Nothing moved. Fix what the message names and retry once. For an occupied "
+            "name held by a captured bundle, the folder is its new edition: `raw accept "
+            "incoming/<bundle> --replaces-bundle <raw-bundle>`. Any other occupied name: rename "
+            "the folder in `incoming/` - such a folder has no replacement form",
+        ),
+        cli_contract.Failure(
+            cause="A file - given directly, inside a folder, or as `--replaces`/`--replaces-bundle` "
+            "material - is a guideline export: its first line, after an optional BOM, starts with "
+            "`<!-- wikitool:export`",
+            reaction="Not fixed by retrying: it is generated from a wiki's `kb/` and never goes "
+            "back into `raw/`. Remove it from `incoming/`; nothing was moved",
+        ),
+        cli_contract.Failure(
+            label="raw accept",
+            cause="`--fidelity`/`--authority` is missing, or names `unknown` or a value outside "
+            "the schema's enum",
+            reaction="Pass both with a valid value, then retry once",
+        ),
+        cli_contract.Failure(
+            label="raw accept",
+            cause="The target name is already occupied anywhere under `raw/` by something the "
+            "call does not own",
+            reaction="Not fixed by retrying: the refusal names `--replaces` (same source, new "
+            "edition) and renaming in `incoming/` (a separate source) as the two routes, and "
+            "neither is the tool's to pick. Show the message to the user and wait",
+        ),
+        cli_contract.Failure(
+            label="raw accept",
+            cause="`--page` names an unknown page or one with no `raw_files:` yet, an existing "
+            "`raw_files:` entry is missing on disk, a file to be moved has more than one owning "
+            "page, or `--page` would overwrite an already-set `fidelity`/`authority` with a "
+            "different value",
+            reaction="Fix the named argument and retry once; a different capture value on an "
+            "existing page is a new edition - `--replaces`",
+        ),
+        cli_contract.Failure(
+            label="raw accept --replaces",
+            cause="More than one incoming file, or `--page` also given",
+            reaction="A replacement is one file for one file - fix the call and retry once",
+        ),
+        cli_contract.Failure(
+            label="raw accept --replaces",
+            cause="The incoming file does not exist or is not directly in `incoming/`, its "
+            "filename differs from the target's, or the target does not lie under `raw/` or "
+            "does not exist",
+            reaction="Fix the named argument and retry once - every check runs before the "
+            "filesystem is touched, so both files are exactly as they were",
+        ),
+        cli_contract.Failure(
+            label="raw accept --replaces",
+            cause="`--fidelity`/`--authority` names `unknown` or a value outside the schema's "
+            "enum, or the target has more than one owning source page",
+            reaction="Fix the named argument and retry once; nothing was touched",
+        ),
+        cli_contract.Failure(
+            label="raw accept --replaces / --page",
+            cause="The target, or a file the page already has, lies in a captured bundle",
+            reaction="Not fixed by retrying: a captured bundle changes only as a whole - `raw "
+            "capture --update <raw-bundle>`, then `raw accept --replaces-bundle`",
+        ),
+        cli_contract.Failure(
+            label="raw accept incoming/<bundle>",
+            cause="A captured folder is given `--fidelity`/`--authority`, its `_capture.json` "
+            "cannot be read or names an invalid value, or its files differ from the ones the "
+            "manifest lists",
+            reaction="Nothing moved. Drop the two flags and retry once; a manifest that does not "
+            "match its folder is fixed by removing the folder and capturing it again, never by "
+            "editing either",
+        ),
+        cli_contract.Failure(
+            label="raw accept --replaces-bundle",
+            cause="Either side has no `_capture.json`, the two manifests name different "
+            "repositories, the folder's name differs from the bundle's, `--page`, `--replaces`, "
+            "`--fidelity` or `--authority` was also given, the folder does not match its "
+            "manifest, a new path is over the budget, or a file of the bundle has more than one "
+            "owning source page",
+            reaction="Nothing under `raw/` or in `incoming/` changed. Fix what the message names "
+            "and retry once; different repositories or a different name are a separate source, "
+            "not a new edition - show the message to the user",
+        ),
+    ),
+    examples=(
+        "tools/wikitool raw accept incoming/docker-cheatsheet.md --fidelity verbatim "
+        "--authority reporting",
+        'tools/wikitool raw accept incoming/part-2.md --fidelity verbatim --authority reporting '
+        '--page "Source - Docker Cheatsheet"',
+        "tools/wikitool raw accept incoming/projekt-x --fidelity verbatim --authority reporting",
+        "tools/wikitool raw accept incoming/cluster.md --replaces raw/documents/cluster.md",
+        "tools/wikitool raw accept incoming/chemenu --replaces-bundle raw/2026/10/chemenu",
+    ),
+    never=(
+        "Never choose the destination under `raw/` by hand, and never move a file into `raw/` "
+        "yourself.",
+        "Never pick between `--replaces` and renaming on your own initiative after a "
+        "name-occupied refusal - the user tells the two intents apart.",
+    ),
+    see_also=(
+        "`raw/CONTRACT.md` \"Getting a file in: incoming/\" - the rules and why",
+        "`wikitool raw pending` - what is waiting in `incoming/`, and which entry is next",
+        "`wikitool raw capture` - writes a captured bundle, or its new edition, into `incoming/`",
+        "`wikitool types describe source` - the capture field values",
+        "`wikitool new source` - the source page for a promoted file",
+    ),
+))
+def raw_accept_command(
+    files: list[Path] = typer.Argument(
+        ...,
+        help="One or more files directly in incoming/, all belonging to the same source - or "
+        "one folder in incoming/, accepted whole as one source",
+    ),
+    fidelity: Optional[str] = typer.Option(
+        None,
+        "--fidelity",
+        help="How faithful the capture is (see `wikitool types describe source`). Required unless "
+        "--replaces is given, where it is the optional, sanctioned way to correct an already-set value",
+    ),
+    authority: Optional[str] = typer.Option(
+        None,
+        "--authority",
+        help="What the material may claim about its subject (see `wikitool types describe source`). "
+        "Required unless --replaces is given, where it is the optional, sanctioned way to correct "
+        "an already-set value",
+    ),
+    page: Optional[str] = typer.Option(
+        None,
+        "--page",
+        help="Extend this existing source page's raw_files: with the promoted file(s), "
+        "folding in its already-promoted file if this raises it past one",
+    ),
+    replaces: Optional[Path] = typer.Option(
+        None,
+        "--replaces",
+        help="Overwrite this existing raw/ file in place with the single incoming/ file passed "
+        "alongside it - the only sanctioned way past the stem-uniqueness rule below",
+    ),
+    dry_run: bool = typer.Option(False, "--dry-run", help="List what would move without writing"),
+    # `Annotated`, unlike the options above, so that a direct call - the way the
+    # test suite calls this function - gets a real `None` here rather than the
+    # truthy `OptionInfo` a plain default would leave behind.
+    replaces_bundle: Annotated[Optional[Path], typer.Option(
+        "--replaces-bundle",
+        help="Replace this captured bundle under raw/ as a whole with the new edition in the "
+        "incoming/ folder passed alongside it (written by raw capture --update)",
+    )] = None,
+):
+    """Promote file(s) or one folder from incoming/ into raw/, computing the
+    destination (date shard, bundle or not, bundle name) instead of taking it
+    as an argument. See raw/CONTRACT.md "Getting a file in: incoming/"."""
+    if not files:
+        fail("Pass at least one file to promote.")
+
+    if replaces_bundle is not None:
+        _replace_bundle(files, replaces_bundle, page, replaces, fidelity, authority, dry_run)
+        return
+
+    resolved = [_resolve(f) for f in files]
+    folders = [p for p in resolved if p.is_dir()]
+    if folders and (len(files) != 1 or page is not None or replaces is not None):
+        fail(escape(
+            f"{rel_path(folders[0])} is a folder, and a folder is accepted alone: one source, "
+            "with no other argument, no --page and no --replaces. Accept it in a call of its own."
+        ))
+
+    if replaces is not None:
+        _replace(files, replaces, page, fidelity, authority, dry_run)
+        return
+
+    if folders and (folders[0] / repo_capture.MANIFEST_NAME).is_file():
+        _accept_captured_folder(folders[0], fidelity, authority, dry_run)
+        return
+
+    fidelity_choices, authority_choices = _capture_choices()
+    if fidelity is None or authority is None:
+        missing = ", ".join(n for n, v in (("--fidelity", fidelity), ("--authority", authority)) if v is None)
+        fail(
+            f"{missing} required - raw accept does not guess how faithful a capture is or what it "
+            f"may claim. --fidelity: {', '.join(fidelity_choices)}. --authority: {', '.join(authority_choices)}."
+        )
+    _check_capture_value("fidelity", fidelity, fidelity_choices)
+    _check_capture_value("authority", authority, authority_choices)
+
+    if folders:
+        _accept_folder(folders[0], fidelity, authority, dry_run)
+        return
+
+    _refusals_fail(_check_files, resolved)
+    _warn_underived(resolved)
+
+    pages = None
+    target_page = None
+    existing_raw_paths: list[Path] = []
+    if page is not None:
+        pages = load_kb_pages(config.KB_DIR)
+        target_page = pages.get(page)
+        if target_page is None:
+            fail(f"No page titled '{page}' found under kb/. Create it first, or omit --page.")
+        existing_rel = source_raw_files(target_page)
+        if not existing_rel:
+            fail(
+                f"'{page}' has no raw_files: yet - omit --page and run "
+                "`wikitool new source --set raw_files=...` for a page's first raw file."
+            )
+        existing_raw_paths = [config.ROOT / p for p in existing_rel]
+        for existing in existing_raw_paths:
+            captured = repo_capture.captured_bundle_of(existing, config.RAW_DIR)
+            if captured is not None:
+                fail(escape(_captured_bundle_message(existing, captured, "--page")))
+        missing = [p for p in existing_raw_paths if not p.is_file()]
+        if missing:
+            fail(
+                f"'{page}' claims raw file(s) that do not exist on disk: "
+                f"{', '.join(rel_path(p) for p in missing)}. Fix raw_files: (see `sources coverage`) "
+                "before promoting more."
+            )
+
+    moves = _refusals_fail(_plan_file_moves, resolved, existing_raw_paths, page)
+
+    moving_existing = [src for src, _dst in moves if src in existing_raw_paths]
+    if moving_existing:
+        by_raw = source_pages_by_raw_file(pages)
+        conflicts = [
+            (rel_path(src), sorted(set(by_raw.get(rel_path(src), [])) - {page}))
+            for src in moving_existing
+        ]
+        conflicts = [(p, owners) for p, owners in conflicts if owners]
+        if conflicts:
+            listed = "\n".join(f"  - {p}: also claimed by {', '.join(o)}" for p, o in conflicts)
+            fail(
+                "Cannot bundle: the following already-covered raw file(s) have more than one "
+                f"owner, so moving them would break the other page(s)' raw_files::\n{listed}\n"
+                "Resolve the multiple ownership first (see `wikitool sources coverage`)."
+            )
+
+    if target_page is not None:
+        current_fidelity = target_page.frontmatter.get("fidelity")
+        current_authority = target_page.frontmatter.get("authority")
+        for field, value, current in (
+            ("fidelity", fidelity, current_fidelity),
+            ("authority", authority, current_authority),
+        ):
+            if current not in (None, "") and current != value:
+                fail(
+                    f"'{page}' already has {field}={current!r} - a capture field is fixed once. "
+                    "Use `raw accept --replaces` to correct it."
+                )
+
+    if dry_run:
+        for src, dst in moves:
+            typer.echo(f"[dry-run] would move {rel_path(src)} -> {rel_path(dst)}")
+        if target_page is not None:
+            moved_map = dict(moves)
+            final = [moved_map.get(p, p) for p in existing_raw_paths] + [moved_map[p] for p in resolved]
+            typer.echo(f"[dry-run] would set raw_files: on '{page}' to {[rel_path(p) for p in final]}")
+            typer.echo(f"[dry-run] would set fidelity={fidelity!r}, authority={authority!r} on '{page}'")
+        typer.echo(f"[dry-run] would move {len(moves)} file(s). No files written.")
+        return
+
+    for src, dst in moves:
+        dst.parent.mkdir(parents=True, exist_ok=True)
+        src.rename(dst)
+        typer.echo(f"  moved {rel_path(src)} -> {rel_path(dst)}")
+
+    moved_map = dict(moves)
+    if target_page is not None:
+        final = [moved_map.get(p, p) for p in existing_raw_paths] + [moved_map[p] for p in resolved]
+        target_page.frontmatter["raw_files"] = [rel_path(p) for p in final]
+        target_page.frontmatter["fidelity"] = fidelity
+        target_page.frontmatter["authority"] = authority
+        write_page(target_page.path, target_page.frontmatter, target_page.body)
+        success(
+            f"Promoted {len(resolved)} file(s); updated raw_files: on '{page}' "
+            f"({len(final)} file(s) total)."
+        )
+        return
+
+    promoted = ", ".join(rel_path(moved_map[p]) for p in resolved)
+    raw_files_arg = ",".join(rel_path(moved_map[p]) for p in resolved)
+    success(
+        f"Promoted {len(resolved)} file(s) to {promoted}.\n"
+        "  Next:\n"
+        "  tools/wikitool new source --name \"<Title>\" \\\n"
+        f"    --set raw_files={raw_files_arg} \\\n"
+        f"    --set fidelity={fidelity} --set authority={authority} \\\n"
+        "    --set source_type=<category>"
+    )

tools/chemenu/commands/raw/capture.py

@@ -0,0 +1,429 @@
+"""`wikitool raw capture` and `raw status` - documentation from a git repository,
+as one bundle per repository and ref rule. The git side - resolving, fetching,
+selecting, the manifest - lives in `chemenu.repo_capture`; these adapters decide
+where the files go and what the caller is told. Like `raw fetch`, `raw capture`
+ends in `incoming/`, never in `raw/`: promoting stays `raw accept`'s job.
+"""
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.markup import escape
+
+from chemenu import cli_contract, config, repo_capture
+from chemenu.commands._util import fail, path_budget_problem_for, rel_path, success
+from chemenu.errors import BackendError, ValidationError
+from chemenu.kb_scan import load_kb_pages
+from chemenu.commands.raw._common import (
+    _capture_choices,
+    _check_capture_value,
+    _folder_collision_message,
+    _incoming_dir,
+    _occupied_stems,
+    _render_changes,
+    _resolve,
+    _shard_dir,
+    _unusable_name,
+)
+
+
+def _check_bundle_name(name: str) -> None:
+    if _unusable_name(name):
+        fail(escape(
+            f"--name {name!r} is not a usable bundle name: one directory name - no path separator, "
+            f"not hidden, not {repo_capture.MANIFEST_NAME}."
+        ))
+
+
+def _captured_raw_bundle(path: Path) -> tuple[Path, repo_capture.Manifest]:
+    bundle = _resolve(path)
+    try:
+        bundle.relative_to(config.RAW_DIR)
+    except ValueError:
+        fail(escape(f"--update {rel_path(bundle)} does not lie under raw/."))
+    manifest_path = bundle / repo_capture.MANIFEST_NAME
+    if not manifest_path.is_file():
+        fail(escape(
+            f"--update {rel_path(bundle)} is not a captured bundle - it has no "
+            f"{repo_capture.MANIFEST_NAME}. Only a bundle raw capture wrote can be updated."
+        ))
+    try:
+        return bundle, repo_capture.read_manifest(manifest_path)
+    except ValidationError as exc:
+        fail(escape(f"{rel_path(manifest_path)}: {exc}"))
+
+
+def _write_bundle(target: Path, files: dict[str, bytes], manifest: repo_capture.Manifest) -> None:
+    """Create `target` with every file and the manifest, or nothing at all:
+    written into a fresh hidden directory beside it (which `raw pending`
+    skips), then renamed into place in one step. A failure removes only that
+    directory, which this call created and nothing else wrote into."""
+    target.parent.mkdir(parents=True, exist_ok=True)
+    staging = Path(tempfile.mkdtemp(prefix=f".{target.name}-", dir=target.parent))
+    try:
+        for rel, data in files.items():
+            path = staging / rel
+            path.parent.mkdir(parents=True, exist_ok=True)
+            with path.open("xb") as handle:
+                handle.write(data)
+        (staging / repo_capture.MANIFEST_NAME).write_text(
+            manifest.to_json(), encoding="utf-8", newline="\n"
+        )
+        os.rename(staging, target)
+    except OSError as exc:
+        shutil.rmtree(staging, ignore_errors=True)
+        fail(escape(f"Could not write {rel_path(target)}/: {exc}. Nothing was left in incoming/."))
+
+
+@cli_contract.record(cli_contract.CommandRecord(
+    path="raw capture",
+    summary="Capture documentation from a git repository into `incoming/<bundle>/`, with a "
+    "manifest naming the repository, ref rule and commit, for `raw accept` to promote.",
+    synopsis=(
+        cli_contract.Variant(
+            usage="raw capture <repo-url> --ref <branch|tag-pattern> --path <glob> [--path <glob> "
+            "...] --name <bundle> --fidelity <v> --authority <v>",
+            notes="First capture of a repository",
+        ),
+        cli_contract.Variant(
+            usage="raw capture --update <raw-bundle> [--fidelity <v>] [--authority <v>]",
+            notes="Capture the current state of an already-accepted bundle, for `raw accept "
+            "--replaces-bundle`",
+        ),
+    ),
+    properties=cli_contract.Properties(
+        effect=cli_contract.Effect.WRITE,
+        idempotent=cli_contract.Idempotent.NO,
+        atomic="Yes for what it leaves in `incoming/` - the bundle is written into a hidden "
+        "staging directory and renamed into place in one step; a failure removes the staging "
+        "directory. The git cache under `tools/.wikitool_capture/` is updated either way",
+        budget=cli_contract.Budget.COUNTED,
+        network=cli_contract.Network.YES,
+    ),
+    notes=(
+        "Writes into `incoming/<bundle>/` only, never into `raw/`: `raw accept incoming/<bundle>` "
+        "promotes it, as one folder, to `raw/<YYYY>/<MM>/<bundle>/`.",
+        "`--ref` is a branch name (`main`) or a tag pattern (`v*`, any of `*?[`), which names the "
+        "newest matching tag by version order. It resolves to one commit - for an annotated tag, "
+        "the commit it points at.",
+        "The commit is fetched by ref name, shallowly, into a bare cache repository per URL under "
+        "`tools/.wikitool_capture/` (gitignored). Files are read from it as blobs, never through a "
+        "checkout, so they are byte-identical to the repository's - no line-ending conversion, no "
+        "filter.",
+        "`--path` globs (repeatable) select files by their path in the repository, as git's "
+        "`:(glob)` pathspec does: `*`, `?` and `[...]` do not cross `/`; `**` as a whole segment "
+        "spans any number of directories; a glob without wildcards also takes everything below it "
+        "as a directory. Files keep their repository paths inside the bundle.",
+        "Never captured, each named in the output with its reason: a file whose first line starts "
+        "with `<!-- wikitool:export`, a symlink, a submodule, a file over 25 MiB, a path with a "
+        "segment starting with `.`, a file named `_capture.json`, and a Git LFS pointer.",
+        "`incoming/<bundle>/_capture.json` records `schema`, `repo`, `ref`, `commit`, `paths`, "
+        "`captured` (UTC), `fidelity`, `authority` and `files`. It is the only declaration that "
+        "and how this instance follows the repository.",
+        "Only `ssh://`, `https://` and the scp form `user@host:path`; a URL with a password or "
+        "token in it is refused, since the manifest is committed. Git runs with the host's own "
+        "credentials and configuration, restricted to those protocols, and never prompts: a "
+        "repository that asks for credentials is reported as unreachable. Each git call times out "
+        "after 120 s.",
+        "`--update <raw-bundle>` takes URL, ref rule and globs from that bundle's manifest and "
+        "writes the current state to `incoming/<same bundle name>/`; `--fidelity`/`--authority` "
+        "default to the manifest's values, and passing one is the way to correct it.",
+        "Refused before anything is written: `incoming/<bundle>` already exists, the name is "
+        "already taken under `raw/` (a first capture), nothing matches the globs, or a target "
+        "path - measured at its later place under `raw/` - is over the path budget.",
+        "Success prints the commit and the `raw accept` line that comes next.",
+    ),
+    failures=(
+        cli_contract.Failure(
+            cause="The URL is not `ssh://`, `https://` or `user@host:path`, or carries a password; "
+            "the `--ref` rule, a `--path` glob or `--name` is unusable; or an argument of the other "
+            "variant was mixed in",
+            reaction="Fix the call and retry once. A credential goes into git's credential helper, "
+            "never into the URL",
+        ),
+        cli_contract.Failure(
+            cause="`--fidelity`/`--authority` is missing on a first capture, or names `unknown` or "
+            "a value outside the schema's enum",
+            reaction="Pass both with a valid value, then retry once",
+        ),
+        cli_contract.Failure(
+            cause="The repository could not be reached, asked for credentials, or timed out; or "
+            "no branch or tag matches `--ref`",
+            reaction="Nothing was written to `incoming/`. Check the URL, the ref rule and the "
+            "host's git credentials with the user; an unreachable host may be retried once",
+        ),
+        cli_contract.Failure(
+            cause="`incoming/<bundle>` already exists, the name is already taken under `raw/`, "
+            "nothing matches the globs, or a path would be over the budget",
+            reaction="Nothing was written. Accept or remove what is in `incoming/`; for a name "
+            "taken by a captured bundle use `--update`; widen or narrow the globs; then retry once",
+        ),
+        cli_contract.Failure(
+            label="raw capture --update",
+            cause="The path is not a bundle under `raw/` with a readable `_capture.json`, or its "
+            "manifest names a URL that is refused",
+            reaction="Show the message to the user - a manifest under `raw/` is never edited by "
+            "hand to make this pass",
+        ),
+    ),
+    examples=(
+        "tools/wikitool raw capture ssh://git@example.org/team/service.git --ref main "
+        "--path 'docs/**/*.md' --path README.md --name service-docs --fidelity verbatim "
+        "--authority normative",
+        "tools/wikitool raw capture https://example.org/team/lib.git --ref 'v*' --path docs "
+        "--name lib-docs --fidelity verbatim --authority reporting",
+        "tools/wikitool raw capture --update raw/2026/10/service-docs",
+    ),
+    never=(
+        "Never put a password or token into the repository URL.",
+        "Never edit a `_capture.json`, or the files of a captured bundle, by hand.",
+        "Never capture a repository the user has not named in this session.",
+    ),
+    see_also=(
+        "`raw/CONTRACT.md` \"Getting a repository in: `raw capture`\" - the rules and why",
+        "`wikitool raw accept` - promotes the bundle; `--replaces-bundle` for a new edition",
+        "`wikitool raw status` - which captured bundles have fallen behind their repository",
+    ),
+))
+def raw_capture_command(
+    repo_url: Optional[str] = typer.Argument(None, help="The repository URL (ssh://, https:// or user@host:path)"),
+    ref: Optional[str] = typer.Option(None, "--ref", help="A branch name (main) or a tag pattern (v*)"),
+    paths: Optional[list[str]] = typer.Option(None, "--path", help="A glob over the repository's paths; repeatable"),
+    name: Optional[str] = typer.Option(None, "--name", help="The bundle name - the folder in incoming/ and under raw/"),
+    fidelity: Optional[str] = typer.Option(None, "--fidelity", help="How faithful the capture is (see `wikitool types describe source`)"),
+    authority: Optional[str] = typer.Option(None, "--authority", help="What the material may claim about its subject (see `wikitool types describe source`)"),
+    update: Optional[Path] = typer.Option(None, "--update", help="Capture the current state of this captured bundle under raw/, from its own manifest"),
+):
+    """Capture documentation from a git repository into incoming/<bundle>/.
+    See raw/CONTRACT.md "Getting a repository in: `raw capture`"."""
+    fidelity_choices, authority_choices = _capture_choices()
+    _check_capture_value("fidelity", fidelity, fidelity_choices)
+    _check_capture_value("authority", authority, authority_choices)
+
+    if update is not None:
+        if repo_url is not None or ref is not None or paths or name is not None:
+            fail("--update takes the URL, --ref, --path and --name from the bundle's manifest - pass "
+                 "none of them alongside it.")
+        bundle, old = _captured_raw_bundle(update)
+        url, rule, globs, bundle_name = old.repo, old.ref, list(old.paths), bundle.name
+        fidelity = fidelity or old.fidelity
+        authority = authority or old.authority
+        target_base = bundle
+    else:
+        missing = [
+            flag for flag, value in (
+                ("<repo-url>", repo_url), ("--ref", ref), ("--path", paths), ("--name", name),
+                ("--fidelity", fidelity), ("--authority", authority),
+            ) if not value
+        ]
+        if missing:
+            fail(
+                f"{', '.join(missing)} required for a first capture (or --update <raw-bundle> for a "
+                f"new edition). --fidelity: {', '.join(fidelity_choices)}. --authority: "
+                f"{', '.join(authority_choices)}."
+            )
+        url, rule, globs, bundle_name = repo_url, ref, list(paths), name
+        _check_bundle_name(bundle_name)
+        target_base = _shard_dir() / bundle_name
+        old = None
+    try:
+        repo_capture.check_repo_url(url)
+        repo_capture.check_ref_rule(rule)
+        for pattern in globs:
+            repo_capture.check_glob(pattern)
+    except ValidationError as exc:
+        fail(escape(str(exc)))
+
+    target = _incoming_dir() / bundle_name
+    if target.exists() or target.is_symlink():
+        fail(escape(
+            f"{rel_path(target)} already exists - raw capture never overwrites. Accept or remove "
+            "what is there first."
+        ))
+    if old is None:
+        holder = _occupied_stems(config.RAW_DIR).get(bundle_name)
+        if holder is not None:
+            fail(escape(_folder_collision_message(bundle_name, holder)))
+
+    try:
+        snap = repo_capture.snapshot(url, rule, globs)
+    except BackendError as exc:
+        fail(escape(f"{url} is not reachable: {exc}. Nothing was written."))
+    except ValidationError as exc:
+        fail(escape(f"{exc} Nothing was written."))
+
+    files = snap.selection.files
+    for path, reason in snap.selection.excluded:
+        typer.echo(f"  excluded {path} ({reason})")
+    if not files:
+        fail(escape(
+            f"Nothing in {url} at {snap.commit[:12]} matches {', '.join(globs)} - after the "
+            "exclusions above, if any. Nothing was written."
+        ))
+    problems = [p for rel in files if (p := path_budget_problem_for(target_base / rel))]
+    if problems:
+        listed = "\n".join(f"  - {p}" for p in problems)
+        fail(escape(
+            f"Cannot capture - these files would be over the path budget at their place under "
+            f"raw/:\n{listed}\n  Narrow the --path globs, or choose a shorter --name. Nothing was written."
+        ))
+
+    manifest = repo_capture.Manifest(
+        repo=url, ref=rule, commit=snap.commit, paths=tuple(globs), captured=repo_capture.utc_now(),
+        fidelity=fidelity, authority=authority, files=tuple(sorted(files)),
+    )
+    _write_bundle(target, files, manifest)
+
+    ref_label = snap.refname.removeprefix("refs/heads/").removeprefix("refs/tags/")
+    lines = [
+        f"Captured {len(files)} file(s) from {url} at {ref_label} ({snap.commit[:12]}) to "
+        f"{rel_path(target)}/."
+    ]
+    if snap.selection.excluded:
+        lines.append(f"  {len(snap.selection.excluded)} matching path(s) excluded - listed above.")
+    if old is not None:
+        if old.commit == snap.commit:
+            lines.append(f"  The commit is the one {rel_path(bundle)}/ already holds.")
+        lines.append(f"  Next: tools/wikitool raw accept {rel_path(target)} --replaces-bundle {rel_path(bundle)}")
+    else:
+        lines.append(f"  Next (after the commitment question in wiki-ingest): tools/wikitool raw accept {rel_path(target)}")
+    success(escape("\n".join(lines)))
+
+
+def _loose_manifest_fields(path: Path) -> dict:
+    """`repo` and `ref` out of a manifest `read_manifest` refused, for the
+    status line - best effort, never raises."""
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+    if not isinstance(data, dict):
+        return {}
+    return {k: data[k] for k in ("repo", "ref", "commit") if isinstance(data.get(k), str)}
+
+
+def _bundle_status(manifest_path: Path) -> dict:
+    bundle = manifest_path.parent
+    row = {
+        "bundle": rel_path(bundle), "repo": None, "ref": None, "old": None, "new": None,
+        "changed": False, "files": [], "error": None,
+    }
+    try:
+        manifest = repo_capture.read_manifest(manifest_path)
+    except ValidationError as exc:
+        loose = _loose_manifest_fields(manifest_path)
+        row.update(repo=loose.get("repo"), ref=loose.get("ref"), old=loose.get("commit"))
+        row["error"] = f"{repo_capture.MANIFEST_NAME} refused: {exc}"
+        return row
+    row.update(repo=manifest.repo, ref=manifest.ref, old=manifest.commit)
+    try:
+        remote = repo_capture.remote_commit(manifest.repo, manifest.ref)
+        row["new"] = remote.commit
+        if remote.commit == manifest.commit:
+            return row
+        snap = repo_capture.snapshot(manifest.repo, manifest.ref, list(manifest.paths))
+    except BackendError as exc:
+        row["error"] = f"not reachable: {exc}"
+        return row
+    except ValidationError as exc:
+        row["error"] = str(exc)
+        return row
+    row["new"] = snap.commit
+    changes = repo_capture.diff(repo_capture.bundle_files(bundle), snap.selection.files)
+    row["files"] = [{"path": rel_path(bundle / rel), "status": status} for status, rel in changes]
+    row["changed"] = bool(changes)
+    return row
+
+
+@cli_contract.record(cli_contract.CommandRecord(
+    path="raw status",
+    summary="Report which captured bundles under `raw/` have fallen behind their repository, "
+    "with each changed file as `A`/`M`/`D`.",
+    synopsis=(cli_contract.Variant(usage="raw status [--json]"),),
+    properties=cli_contract.Properties(
+        effect=cli_contract.Effect.READ,
+        idempotent=cli_contract.Idempotent.YES,
+        atomic="Read-only for the tree - writes nothing but the git cache under "
+        "`tools/.wikitool_capture/`",
+        budget=cli_contract.Budget.COUNTED,
+        network=cli_contract.Network.YES,
+    ),
+    notes=(
+        "Reads every `_capture.json` under `raw/` and resolves its ref rule with `git ls-remote`. "
+        "Only where the commit moved is the new commit fetched, and compared - with the same globs "
+        "and exclusions `raw capture` applies - against the bundle's files under `raw/`.",
+        "A bundle whose commit moved without a change inside its globs is reported as unchanged, "
+        "not listed. A tag pattern follows new matching tags only, never new commits on a branch.",
+        "Each changed bundle prints `old -> new` and its files as `A`/`M`/`D`, grouped by owning "
+        "source page with its citing pages, followed by the `raw capture --update` and `raw accept "
+        "--replaces-bundle` lines that take the new edition in.",
+        "An unreachable repository, a refused URL or an unreadable manifest is one line beside the "
+        "others, never an abort; the exit status is 0 as long as the command itself ran.",
+        "`--json` prints one object per bundle: `bundle`, `repo`, `ref`, `old`, `new`, `changed`, "
+        "`files` (`[{path, status}]`), `error` (or null).",
+        "Git runs exactly as for `raw capture`: the host's credentials, only `ssh`/`https`, never "
+        "a prompt, 120 s per call.",
+    ),
+    failures=(
+        cli_contract.Failure(
+            code=0,
+            cause="A repository could not be reached, its manifest or URL was refused, or no ref "
+            "matches its rule",
+            reaction="Reported on that bundle's line, the others are still checked; check the URL "
+            "and the host's git credentials with the user",
+        ),
+    ),
+    examples=(
+        "tools/wikitool raw status",
+        "tools/wikitool raw status --json",
+    ),
+    never=(
+        "Never edit a `_capture.json` to silence a line - a new edition goes through `raw capture "
+        "--update` and `raw accept --replaces-bundle`.",
+    ),
+    see_also=(
+        "`wikitool raw capture` - `--update` captures the new edition",
+        "`wikitool raw accept` - `--replaces-bundle` takes it in",
+        "`raw/CONTRACT.md` \"Getting a repository in: `raw capture`\"",
+    ),
+))
+def raw_status_command(
+    json_out: bool = typer.Option(False, "--json", help="Print one JSON object per captured bundle"),
+):
+    """Report which captured bundles have fallen behind their repository.
+    See raw/CONTRACT.md "Getting a repository in: `raw capture`"."""
+    rows = [_bundle_status(p) for p in repo_capture.captured_manifests(config.RAW_DIR)]
+
+    if json_out:
+        typer.echo(json.dumps(rows, indent=2, ensure_ascii=False))
+        return
+    if not rows:
+        typer.echo("No captured bundle under raw/.")
+        return
+
+    pages = load_kb_pages(config.KB_DIR) if any(r["changed"] for r in rows) else {}
+    unchanged = 0
+    typer.echo(f"{len(rows)} captured bundle(s) under raw/:")
+    for row in rows:
+        if row["error"]:
+            typer.echo(f"  {row['bundle']}  {row['repo'] or '?'}  {row['ref'] or '?'}  - {row['error']}")
+            continue
+        if not row["changed"]:
+            unchanged += 1
+            continue
+        typer.echo(
+            f"  {row['bundle']}  {row['repo']}  {row['ref']}  {row['old'][:12]} -> {row['new'][:12]}"
+        )
+        changes = [(f["status"], f["path"]) for f in row["files"]]
+        for line in _render_changes(changes, pages, "    "):
+            typer.echo(line)
+        name = Path(row["bundle"]).name
+        typer.echo(f"    Next: tools/wikitool raw capture --update {row['bundle']}")
+        typer.echo(f"          tools/wikitool raw accept incoming/{name} --replaces-bundle {row['bundle']}")
+    typer.echo(f"{unchanged} unchanged.")

tools/chemenu/commands/raw/derive.py

@@ -0,0 +1,500 @@
+"""`wikitool raw derive` - the sanctioned intake for a document that is not text: a
+PDF, a mail, an Office file. Like `raw fetch` it ends in `incoming/`, never in
+`raw/`. The formats live in `chemenu.filters`; this adapter decides names and
+folders and what the caller is told. Everything is read, derived and checked
+before the first write, so a refusal leaves `incoming/` as it was.
+
+`raw derive` is the way in for a document that is not text (PDF, mail, Office
+file): it turns one file in `incoming/` into a folder `<stem>/` holding the
+untouched `original.<ext>` and a `derived.md` made by the registered filter
+(`chemenu.filters`), so `raw accept` takes the folder as one source. Everything
+is checked before the first write; the original moves in last.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+from rich.markup import escape
+
+from chemenu import cli_contract, config, filters, repo_capture
+from chemenu.commands._util import fail, path_budget_problem_for, rel_path, success
+from chemenu.errors import ValidationError
+from chemenu.commands.raw._common import (
+    _Refused,
+    _check_directly_in_incoming,
+    _folder_collision_message,
+    _incoming_dir,
+    _occupied_stems,
+    _refusals_fail,
+    _resolve,
+    _shard_dir,
+    _unusable_name,
+    _walk_folder,
+    _write_exclusive,
+)
+
+
+DERIVE_MAX_BYTES = 100 * 1024 * 1024
+ORIGINAL_STEM = "original"
+DERIVED_NAME = "derived.md"
+
+
+@dataclass(frozen=True)
+class _Derived:
+    """One finished derivation, not yet written."""
+
+    filter: filters.Filter
+    derivation: filters.Derivation
+    source: Path  # the file in incoming/
+    markdown: bytes
+    target: Path  # where the .md goes
+    extras: tuple[tuple[Path, bytes], ...] = ()  # the derivation's further files, beside the .md
+
+
+def _read_for_derive(path: Path) -> bytes:
+    size = path.stat().st_size
+    if size > DERIVE_MAX_BYTES:
+        raise _Refused(
+            f"{rel_path(path)} is {size // (1024 * 1024)} MiB; raw derive reads at most "
+            f"{DERIVE_MAX_BYTES // (1024 * 1024)} MiB."
+        )
+    return path.read_bytes()
+
+
+def _run_filter(candidate: filters.Filter, data: bytes, path: Path) -> filters.Derivation:
+    try:
+        filters.require_available(candidate)
+        return candidate.derive(data, path.name)
+    except ValidationError as exc:
+        raise _Refused(f"{rel_path(path)}: {exc}") from None
+
+
+def _collision_warning(name: str) -> Optional[str]:
+    holder = _occupied_stems(config.RAW_DIR).get(name)
+    if holder is None:
+        return None
+    return _folder_collision_message(name, holder)
+
+
+@dataclass(frozen=True)
+class _FilePlan:
+    """A single file turned into a folder, not yet written."""
+
+    folder: Path
+    stem: str
+    candidate: filters.Filter
+    original: Optional[Path]  # where the source moves to; None when the source is only unpacked
+    derived: Optional[Path]
+    markdown: bytes
+    container: Optional[filters.Container]
+    derivations: list[filters.Derivation]
+    extras: tuple[tuple[str, bytes], ...] = ()  # (path in the folder, bytes) of the derivation's further files
+
+
+def _plan_derive_file(path: Path, name: Optional[str]) -> _FilePlan:
+    """The single-file form: `incoming/<stem>/` with the original moved in as
+    `original.<ext>` and the derived text beside it - or, for a container (a ZIP,
+    a mail), the members under their own names."""
+    incoming = _incoming_dir()
+    if not path.is_file():
+        raise _Refused(f"{rel_path(path)} does not exist or is not a file.")
+    _check_directly_in_incoming(path, incoming)
+    if path.name.startswith(".") or path.is_symlink():
+        raise _Refused(f"{rel_path(path)} is hidden or a symlink - raw derive does not take it.")
+    candidate = filters.filter_for(path.name)
+    if candidate is None:
+        listed = ", ".join(filters.supported_extensions())
+        raise _Refused(
+            f"No filter reads {filters.extension_of(path.name) or 'a file without an extension'} "
+            f"({rel_path(path)}). raw derive reads: {listed}. Any other file is accepted as it is "
+            "- note what it shows."
+        )
+    stem = name if name is not None else path.stem
+    if _unusable_name(stem):
+        raise _Refused(
+            f"{stem!r} is not a usable folder name: one directory name - no path separator, not "
+            f"hidden, not {repo_capture.MANIFEST_NAME}. Pass --name <stem>."
+        )
+    folder = incoming / stem
+    if os.path.lexists(folder):
+        raise _Refused(
+            f"{rel_path(folder)} already exists in incoming/ - raw derive never overwrites. Accept "
+            "or remove what is there, or pass --name <stem>."
+        )
+    extension = filters.extension_of(path.name)
+    data = _read_for_derive(path)
+    container = None
+    try:
+        filters.require_available(candidate)
+        container = candidate.container(data, path.name)
+    except ValidationError as exc:
+        raise _Refused(f"{rel_path(path)}: {exc}") from None
+    if container is not None and container.text is None:
+        original = derived = None
+        markdown = b""
+        targets = [rel for rel, _data in container.files]
+        derivations = list(container.derivations)
+        extras = ()
+    else:
+        original = folder / f"{ORIGINAL_STEM}{extension}"
+        derived = folder / DERIVED_NAME
+        derivation = container.text if container is not None else _run_filter(candidate, data, path)
+        assert derivation is not None
+        companion = filters.companion_name(DERIVED_NAME) if derivation.files else None
+        markdown = filters.render(
+            candidate, derivation, derived_from=original.name, original_name=path.name,
+            companion=companion,
+        )
+        extras = tuple((f"{companion}/{name}", content) for name, content in derivation.files)
+        targets = [original.name, derived.name] + [rel for rel, _data in extras] + (
+            [rel for rel, _data in container.files] if container is not None else []
+        )
+        derivations = [derivation] + (list(container.derivations) if container is not None else [])
+    for target in targets:
+        problem = path_budget_problem_for(_shard_dir() / stem / target)
+        if problem:
+            raise _Refused(
+                f"Cannot write {problem}. The folder name comes from the file's name: pass "
+                "--name <shorter stem>."
+            )
+    return _FilePlan(folder, stem, candidate, original, derived, markdown, container, derivations, extras)
+
+
+def _write_folder(source: Path, plan: _FilePlan) -> None:
+    """Create the folder and write everything into it, then move the original in
+    (or, for an unpacked archive, remove it) - last, so a failure before it has
+    taken nothing from `incoming/`. A failure part-way removes the folder this
+    call made and puts the original back."""
+    moved = False
+    try:
+        plan.folder.mkdir()
+        if plan.derived is not None:
+            with plan.derived.open("xb") as handle:
+                handle.write(plan.markdown)
+        for rel, data in plan.extras:
+            target = plan.folder / rel
+            target.parent.mkdir(parents=True, exist_ok=True)
+            with target.open("xb") as handle:
+                handle.write(data)
+        if plan.container is not None:
+            for rel, data in plan.container.files:
+                target = plan.folder / rel
+                target.parent.mkdir(parents=True, exist_ok=True)
+                with target.open("xb") as handle:
+                    handle.write(data)
+        if plan.original is not None:
+            source.rename(plan.original)
+            moved = True
+        else:
+            source.unlink()
+    except OSError as exc:
+        if moved and plan.original is not None:
+            plan.original.rename(source)
+        shutil.rmtree(plan.folder, ignore_errors=True)
+        fail(escape(f"Could not write {rel_path(plan.folder)}: {exc}. incoming/ was left as it was."))
+
+
+def _plan_derive_folder(folder: Path):
+    """The folder form: a `<stem>.md` beside every file that has a filter."""
+    incoming = _incoming_dir()
+    try:
+        rel = folder.relative_to(incoming)
+    except ValueError:
+        raise _Refused(
+            f"{rel_path(folder)} is not under incoming/ - raw derive only takes material from there."
+        ) from None
+    if len(rel.parts) != 1:
+        raise _Refused(
+            f"incoming/{rel.as_posix()} is not directly in incoming/ - a folder is derived from "
+            f"the top: raw derive incoming/{rel.parts[0]}"
+        )
+    if folder.name.startswith(".") or folder.is_symlink():
+        raise _Refused(f"{rel_path(folder)} is hidden or a symlink - raw derive does not take it.")
+    if (folder / repo_capture.MANIFEST_NAME).is_file():
+        raise _Refused(f"{rel_path(folder)}/ is a captured bundle; a captured bundle is not derived.")
+    files, _dirs, refused = _walk_folder(folder)
+    if refused:
+        listed = "\n".join(f"  - {entry}" for entry in refused)
+        raise _Refused(
+            f"{rel_path(folder)}/ holds entries raw accept would not take - hidden entries, "
+            f"symlinks and special files:\n{listed}\n  Remove or replace them in incoming/, then "
+            "derive the folder again."
+        )
+    plan: list[tuple[Path, filters.Filter, Path]] = []
+    untouched: list[Path] = []
+    for file in files:
+        candidate = filters.filter_for(file.name)
+        if candidate is None or candidate.container_only:
+            untouched.append(file)
+        else:
+            plan.append((file, candidate, file.with_suffix(".md")))
+    if not plan:
+        raise _Refused(
+            f"No file in {rel_path(folder)}/ has a filter. raw derive reads: "
+            f"{', '.join(filters.supported_extensions())}."
+        )
+    targets: dict[Path, Path] = {}
+    for file, _candidate, md in plan:
+        if md in targets:
+            raise _Refused(
+                f"{rel_path(targets[md])} and {rel_path(file)} would both derive to {rel_path(md)}. "
+                "Rename one of them in incoming/ and derive the folder again."
+            )
+        targets[md] = file
+        if os.path.lexists(md):
+            raise _Refused(
+                f"{rel_path(md)} already exists - raw derive never overwrites. Remove it or "
+                "rename it, then derive the folder again."
+            )
+        problem = path_budget_problem_for(_shard_dir() / folder.name / md.relative_to(folder))
+        if problem:
+            raise _Refused(
+                f"Cannot write {problem}. The path comes from the folder in incoming/: shorten the "
+                "folder's name or the names inside it."
+            )
+    derived: list[_Derived] = []
+    for file, candidate, md in plan:
+        derivation = _run_filter(candidate, _read_for_derive(file), file)
+        companion = md.with_name(filters.companion_name(md.name))
+        extras: tuple[tuple[Path, bytes], ...] = ()
+        if derivation.files:
+            if os.path.lexists(companion):
+                raise _Refused(
+                    f"{rel_path(companion)} already exists - raw derive never overwrites. Remove it "
+                    "or rename it, then derive the folder again."
+                )
+            extras = tuple((companion / name, content) for name, content in derivation.files)
+            for target, _content in extras:
+                problem = path_budget_problem_for(_shard_dir() / folder.name / target.relative_to(folder))
+                if problem:
+                    raise _Refused(
+                        f"Cannot write {problem}. The path comes from the folder in incoming/: "
+                        "shorten the folder's name or the names inside it."
+                    )
+        markdown = filters.render(
+            candidate, derivation, derived_from=file.name, companion=companion.name
+        )
+        derived.append(_Derived(candidate, derivation, file, markdown, md, extras))
+    return derived, untouched
+
+
+def _echo_notes(
+    derivations: list[filters.Derivation], container: Optional[filters.Container] = None
+) -> None:
+    warnings = [w for d in derivations for w in d.warnings]
+    checks = [c for d in derivations for c in d.checks]
+    if container is not None:
+        warnings += container.warnings
+        checks += container.checks
+    for warning in filters.unique(warnings):
+        typer.echo(f"WARN {warning}")
+    for line in filters.unique(checks):
+        typer.echo(f"CHECK {line}")
+
+
+def _container_report(container: filters.Container) -> list[str]:
+    lines = [f"  {src} -> {md} ({name})" for src, md, name in container.derived]
+    if container.plain:
+        lines.append("  No filter for, written as they are: " + ", ".join(container.plain))
+    if container.renamed:
+        lines.append(
+            "  Renamed (a name was taken or reserved): "
+            + ", ".join(f"{old} -> {new}" for old, new in container.renamed)
+        )
+    if container.hidden:
+        lines.append("  Hidden entries, named but not written: " + ", ".join(container.hidden))
+    return lines
+
+
+def _derive_next(target: Path) -> str:
+    return (
+        "  Next (after the commitment question in wiki-ingest):\n"
+        f"  tools/wikitool raw accept --fidelity <value> --authority <value> {rel_path(target)}"
+    )
+
+
+@cli_contract.record(cli_contract.CommandRecord(
+    path="raw derive",
+    summary="Derive reading text from a PDF, e-mail, Office file or web page in `incoming/`, or "
+    "unpack a ZIP, and leave the original beside it, for `raw accept` to promote.",
+    synopsis=(
+        cli_contract.Variant(
+            usage="raw derive incoming/<file> [--name <stem>]",
+            notes="Move the file into the new folder `incoming/<stem>/` as `original.<ext>` and "
+            "write `incoming/<stem>/derived.md`",
+        ),
+        cli_contract.Variant(
+            usage="raw derive incoming/<folder>",
+            notes="Write a `<stem>.md` beside every file in the folder that has a filter",
+        ),
+    ),
+    properties=cli_contract.Properties(
+        effect=cli_contract.Effect.WRITE,
+        idempotent=cli_contract.Idempotent.NO,
+        atomic="Yes for what it leaves behind - everything is read, derived and checked before "
+        "the first write; the files are created exclusively, and a failure part-way puts back "
+        "what this call changed",
+        budget=cli_contract.Budget.COUNTED,
+        network=cli_contract.Network.NO,
+    ),
+    notes=(
+        "Writes into `incoming/` only, never into `raw/`, and reads no network: `raw accept` "
+        "promotes the folder afterwards as one source under `raw/<YYYY>/<MM>/<stem>/`.",
+        "A single file directly in `incoming/` becomes a folder named after its stem (or "
+        "`--name`) holding `original.<ext>` - the original bytes, moved - and `derived.md`. The "
+        "fixed names keep the path inside the path budget; the header of `derived.md` records "
+        "the file's own name as `original_name`.",
+        "A folder is derived in place: each file with a filter gets `<stem>.md` beside it, "
+        "nothing is renamed, and the files without a filter are named in the output.",
+        "`derived.md` starts with a fixed header (`derived_by`, `filter`, `engine`, `derived`, "
+        "`derived_from`, `original_name` for a renamed file, `title`, then the fields of the "
+        "format) followed by the derived text. The same bytes always give the same text; only "
+        "`derived` differs between runs. `engine` names the tool version and, where a package "
+        "reads the format, its version.",
+        "Formats: web pages (`.html`), Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`, "
+        "`.xlsm`), e-mail (`.eml`) and ZIP archives (`.zip`), read with the standard library "
+        "only; PDF (`.pdf`, `pypdf`) and Outlook messages (`.msg`, `olefile`), each read by a "
+        "package in `tools/requirements.txt` that is imported only when such a file is derived. "
+        "A missing package refuses that format alone, with the preflight hint; `wikitool "
+        "doctor` shows one `import-filter-<name>` line per package-backed format. A PDF gets "
+        "`<!-- page N -->` markers and a WARN when it yields almost no text (a scan; there is "
+        "no OCR); an encrypted PDF is refused. A `.msg` is a container like an `.eml`; an "
+        "embedded message inside it is named, not read. The text of a mail (`.eml` and `.msg` "
+        "alike) is its plain text, unless that is under 10% of the length of the HTML text - a "
+        "\"plain text version not available\" placeholder - in which case it is the HTML text "
+        "and a WARN names both lengths. The other version of the text in a "
+        "`multipart/alternative` is not an attachment.",
+        "A workbook (`.xlsx`, `.xlsm`) gets its cells as files: beside its `.md` a folder "
+        "`<md stem>.sheets/` (`derived.sheets/` for a single file) holds one `NN-<sheet>.tsv` "
+        "per sheet - tab-separated per RFC 4180, UTF-8, row 1 of the file is row 1 of the sheet "
+        "- and, for a sheet with formulas, `NN-<sheet>.formulas.tsv` listing cell, formula and "
+        "last stored value. The `.md` is the overview: one section per sheet with its range, "
+        "the file names and a preview. Formulas are not recalculated; a shared formula is "
+        "written out per cell, and one that cannot be is named `shared:<master cell>` with a "
+        "WARN. The folder counts against the path budget and is accepted with the rest; an "
+        "existing one is never overwritten.",
+        "A ZIP and a mail are containers. A ZIP at the top of `incoming/` becomes the folder "
+        "`incoming/<stem>/` holding its files under their own names, each with a `<stem>.md` "
+        "where a filter reads it; the ZIP is removed and the output names it with its sha256. A "
+        "mail becomes a folder with `original.eml`, `derived.md` and its attachments under their "
+        "own names; a ZIP attached to it is unpacked once into a folder of its own, and an "
+        "attached mail is quoted in `derived.md`, not unpacked. A ZIP inside a ZIP, an "
+        "absolute or `..` path, a symbolic link, an encrypted entry, more than 1000 entries, a "
+        "file over 25 MiB or more than 250 MiB in total refuse the whole call. Hidden entries "
+        "(`__MACOSX/`, `.DS_Store`, anything starting with a dot) are named, not written. A name "
+        "taken twice, or reserved (`original.*` and `derived.md` at the top of a mail's folder, "
+        "`_capture.json` anywhere), gets `-2`, `-3` before its extension, and the output lists "
+        "each rename.",
+        "A name already taken under `raw/` is a WARN, not a refusal: whether the folder is a "
+        "later edition of that source or a separate one is the human's call. A later edition of "
+        "a derived source is replaced file by file with `raw accept --replaces`; there is no "
+        "`--replaces-bundle` for it.",
+        "After a success the output prints the warnings the filters raised, at most three "
+        "`CHECK` lines per format that was derived in this run - what to look at before the "
+        "text is trusted - and the `raw accept` line for the result.",
+        "Never edit `derived.md` by hand: a better derivation is a new `raw derive` of the original.",
+    ),
+    failures=(
+        cli_contract.Failure(
+            cause="The path is not directly in `incoming/`, does not exist, or no filter reads "
+            "the file's extension",
+            reaction="Nothing was written. Fix the argument and retry once - a file in a "
+            "subfolder is moved up, or its folder is derived whole",
+        ),
+        cli_contract.Failure(
+            cause="The target folder, a `.md` or a workbook's `.sheets` folder already exists, or the path would be over the "
+            "path budget",
+            reaction="Nothing was written. Remove what is there or pass `--name <shorter stem>`, "
+            "then retry once. An existing `.md` is never overwritten",
+        ),
+        cli_contract.Failure(
+            cause="The document cannot be read (damaged, encrypted), a container breaks one of its "
+            "rules (a ZIP inside a ZIP, an unsafe path, a limit) or the filter's package is not "
+            "installed",
+            reaction="Nothing was written. A missing package is reported with the preflight "
+            "hint - show it to the user and stop, never install it. Any other filter keeps "
+            "working",
+        ),
+    ),
+    examples=(
+        "tools/wikitool raw derive incoming/report.docx",
+        "tools/wikitool raw derive incoming/Angebot-XYZ.xlsx --name angebot-xyz",
+        "tools/wikitool raw derive incoming/meeting-files",
+        "tools/wikitool raw derive incoming/scans.zip",
+    ),
+    never=(
+        "Never edit `derived.md` or its header by hand.",
+        "Never derive a file the user did not put into `incoming/` for this purpose.",
+    ),
+    see_also=(
+        "`raw/CONTRACT.md` \"Getting a document in: `raw derive`\" - the rules and why",
+        "`wikitool raw accept` - promotes the folder into `raw/`",
+        "`instructions/wiki-ingest/SKILL.md` - where a PDF, mail or Office file to ingest starts",
+    ),
+))
+def raw_derive_command(
+    path: Path = typer.Argument(..., help="A file or a folder directly in incoming/"),
+    name: Annotated[Optional[str], typer.Option(
+        "--name", help="Folder stem for a single file instead of the file's own stem"
+    )] = None,
+):
+    """Derive reading text from a document in incoming/ and keep the original.
+    See raw/CONTRACT.md "Getting a document in: `raw derive`"."""
+    resolved = _resolve(path)
+    if resolved.is_dir():
+        if name is not None:
+            fail("--name applies to a single file; a folder keeps its own name.")
+        derived, untouched = _refusals_fail(_plan_derive_folder, resolved)
+        _write_exclusive([write for d in derived for write in [(d.target, d.markdown), *d.extras]])
+        lines = [f"Derived {len(derived)} file(s) in {rel_path(resolved)}/:"]
+        for d in derived:
+            more = f" + {len(d.extras)} file(s) in {rel_path(d.extras[0][0].parent)}/" if d.extras else ""
+            lines.append(f"  {rel_path(d.source)} -> {rel_path(d.target)} ({d.filter.name}){more}")
+        if untouched:
+            lines.append(
+                "  No filter for, left as they are: " + ", ".join(rel_path(p) for p in untouched)
+            )
+        success(escape("\n".join(lines)))
+        warning = _collision_warning(resolved.name)
+        if warning:
+            typer.echo("WARN " + warning)
+        _echo_notes([d.derivation for d in derived])
+        typer.echo(_derive_next(resolved))
+        return
+
+    plan = _refusals_fail(_plan_derive_file, resolved, name)
+    _write_folder(resolved, plan)
+    container = plan.container
+    if plan.original is None:
+        success(escape(
+            f"Unpacked {resolved.name} into {rel_path(plan.folder)}/ ({plan.candidate.name}); "
+            f"{resolved.name} is removed."
+        ))
+    elif container is not None and container.files:
+        success(escape(
+            f"Derived {rel_path(plan.derived)} from {resolved.name} ({plan.candidate.name}) with "
+            f"{len(container.files)} file(s) from its attachments; the original is now "
+            f"{rel_path(plan.original)}."
+        ))
+    else:
+        success(escape(
+            f"Derived {rel_path(plan.derived)} from {resolved.name} ({plan.candidate.name}); the "
+            f"original is now {rel_path(plan.original)}."
+        ))
+    if plan.extras:
+        lines = f"  + {len(plan.extras)} file(s) beside it in {rel_path(plan.folder)}/{plan.extras[0][0].rsplit('/', 1)[0]}/"
+        typer.echo(escape(lines))
+    if container is not None:
+        report = _container_report(container)
+        if report:
+            typer.echo("\n".join(report))
+    warning = _collision_warning(plan.stem)
+    if warning:
+        typer.echo("WARN " + warning)
+    _echo_notes(plan.derivations, container)
+    typer.echo(_derive_next(plan.folder))

tools/chemenu/commands/raw/fetch.py

@@ -0,0 +1,258 @@
+"""`wikitool raw fetch` - the sanctioned intake for a URL the user names. It ends
+in `incoming/`, never in `raw/`: promoting stays `raw accept`'s job, so
+`wiki-ingest` keeps its order (the commitment question comes before the promotion)
+and the capture fields are asked exactly once, there. The fetch and derivation
+themselves live in `chemenu.web_capture`; this adapter decides file names and what
+the caller is told.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.markup import escape
+
+from chemenu import cli_contract, web_capture
+from chemenu.commands._util import fail, rel_path, success
+from chemenu.errors import ChemenuError
+from chemenu.version import VersionError, read_version
+from chemenu.commands.raw._common import (
+    _incoming_dir,
+    _resolve,
+    _validate_under_incoming,
+    _write_exclusive,
+)
+
+
+_TEASER_HINT = (
+    "  If the text stops at a teaser (paywall, login, script-rendered page): save the page from a\n"
+    "  logged-in browser to incoming/ and run  tools/wikitool raw fetch --html "
+    "incoming/<file>.html --url {url}"
+)
+
+
+def _user_agent() -> str:
+    try:
+        version = str(read_version())
+    except VersionError:
+        version = "unknown"
+    return f"chemenu-wikitool/{version} (raw fetch)"
+
+
+def _next_lines(files: list[Path], source_url: str) -> str:
+    paths = " ".join(rel_path(f) for f in files)
+    return (
+        "  Next (after the commitment question in wiki-ingest):\n"
+        f"  tools/wikitool raw accept --fidelity published --authority <value> {paths}\n"
+        f"  and on the source page: --set source_url={source_url}"
+    )
+
+
+def _refuse_existing(targets: list[Path]) -> None:
+    existing = [t for t in targets if t.exists()]
+    if existing:
+        fail(escape(
+            f"{', '.join(rel_path(t) for t in existing)} already exists in incoming/ - raw fetch "
+            "never overwrites. Accept or remove the file that is there, or pass --name <stem>."
+        ))
+
+
+def _warn(derivation: web_capture.Derivation, md_path: Path) -> None:
+    if derivation.decoded.replaced:
+        typer.echo(
+            f"WARN {derivation.decoded.replaced} undecodable byte(s) under "
+            f"{derivation.decoded.charset} were replaced with U+FFFD in {rel_path(md_path)} - "
+            "the .html beside it keeps the original bytes."
+        )
+    if derivation.text_chars < web_capture.SHORT_TEXT_CHARS:
+        typer.echo(
+            f"WARN The derived text is only {derivation.text_chars} characters - likely a "
+            "script-rendered page, a login wall or an empty response. Read "
+            f"{rel_path(md_path)} before accepting it."
+        )
+
+
+def _fetch_html_file(html: Path, source_url: Optional[str]) -> None:
+    if source_url is None:
+        fail("--html needs --url <url>: it goes into the header, resolves relative links, and "
+             "becomes source_url on the source page.")
+    try:
+        web_capture.check_url(source_url)
+    except ChemenuError as exc:
+        fail(escape(str(exc)))
+    path = _resolve(html)
+    _validate_under_incoming(path, _incoming_dir())
+    if not path.is_file():
+        fail(f"{rel_path(path)} does not exist or is not a file.")
+    md_path = path.with_suffix(".md")
+    if md_path == path:
+        fail(f"{rel_path(path)} is already a .md file - --html takes the saved HTML page.")
+    _refuse_existing([md_path])
+
+    derivation = web_capture.derive_document(path.read_bytes(), source_url, path.name, url=source_url)
+    _write_exclusive([(md_path, derivation.markdown)])
+    _warn(derivation, md_path)
+    success(escape(
+        f"Derived {rel_path(md_path)} from {rel_path(path)} (no network access).\n"
+        + _next_lines([path, md_path], source_url)
+    ))
+
+
+@cli_contract.record(cli_contract.CommandRecord(
+    path="raw fetch",
+    summary="Capture a web page the user names into `incoming/`: the HTML as received plus a "
+    "derived text, for `raw accept` to promote.",
+    synopsis=(
+        cli_contract.Variant(
+            usage="raw fetch <url> [--name <stem>]",
+            notes="Fetch the page and write `incoming/<stem>.html` and `incoming/<stem>.md`",
+        ),
+        cli_contract.Variant(
+            usage="raw fetch --html incoming/<file>.html --url <url>",
+            notes="Derive the `.md` from a page a human saved from their browser - no network "
+            "access",
+        ),
+    ),
+    properties=cli_contract.Properties(
+        effect=cli_contract.Effect.WRITE,
+        idempotent=cli_contract.Idempotent.NO,
+        atomic="Yes for what it leaves behind - one or two new files in `incoming/`, created "
+        "exclusively; a failure on the second removes the first",
+        budget=cli_contract.Budget.COUNTED,
+        network=cli_contract.Network.YES,
+    ),
+    notes=(
+        "Writes into `incoming/` only, never into `raw/`: `raw accept` promotes both files "
+        "afterwards, in one call, as one bundle under `raw/<YYYY>/<MM>/<stem>/`.",
+        "The `.html` holds the response body byte for byte. The `.md` starts with a fixed "
+        "header block (`fetched_by`, `url`, `final_url`, `retrieved`, `http_status`, "
+        "`content_type`, `charset`, `title`, `derived_from`) followed by the derived text.",
+        "Character set, in this order: byte-order mark, the HTTP `Content-Type` charset, a "
+        "`<meta>` declaration in the first 4 KiB, UTF-8. Undecodable bytes are replaced with "
+        "U+FFFD and reported; the header names the charset and where it came from.",
+        "Derived text: the content root is `<main>`, else the single `<article>`, else "
+        "`<body>`; `script`, `style`, `noscript`, `nav`, `header`, `footer`, `aside`, `form`, "
+        "`template` and `svg` are dropped below it. Headings, lists, code, tables and links "
+        "(made absolute) become Markdown. The same bytes always give the same text.",
+        "The stem is the URL's last path segment without its extension, else its host name - "
+        "ASCII, lowercase, `-`-separated, at most 60 characters; `--name` overrides it.",
+        "A `text/plain` or `text/markdown` response is stored as received as `.txt`/`.md`; any "
+        "other non-HTML response (PDF, image, ...) as received under the extension of its "
+        "content type. Neither gets a derived file or a header.",
+        "Only `http`/`https`, on every redirect hop too. 30 s for the whole transfer, 25 MiB at "
+        "most. No cookies, no JavaScript: a derived text under 200 characters is written, with "
+        "a warning to read it before accepting.",
+        "`--html` derives the `.md` beside a page saved to `incoming/` from a logged-in browser "
+        "- the way past a paywall or a script-rendered page. Its header carries `derived` (when "
+        "the text was derived) instead of `retrieved`, `final_url`, `http_status` and "
+        "`content_type`, and the `.html` is left untouched.",
+        "Success prints the `raw accept` line for the written files and the `source_url` for the "
+        "source page.",
+    ),
+    failures=(
+        cli_contract.Failure(
+            cause="The URL is not `http`/`https` (also after a redirect), or neither or both of "
+            "`<url>` and `--html` were given",
+            reaction="Fix the call and retry once. A local file is dropped into `incoming/` by "
+            "hand, not fetched",
+        ),
+        cli_contract.Failure(
+            cause="A target file already exists in `incoming/`",
+            reaction="Nothing was written or overwritten. Accept or remove what is there, or "
+            "pass `--name <stem>`, then retry once",
+        ),
+        cli_contract.Failure(
+            cause="The server answered with an HTTP error, could not be reached, took longer "
+            "than 30 s, or sent more than 25 MiB",
+            reaction="Nothing was written. An HTTP 4xx is not fixed by retrying - check the URL "
+            "with the user; for a paywall or login, save the page in a browser and use "
+            "`--html`. An unreachable host or a timeout may be retried once",
+        ),
+        cli_contract.Failure(
+            label="raw fetch --html",
+            cause="`--url` is missing, the file is not directly in `incoming/` or does not "
+            "exist, or `incoming/<stem>.md` already exists",
+            reaction="Fix the named argument and retry once - nothing was written. A saved page "
+            "inside a subdirectory of `incoming/` is moved up into `incoming/` first",
+        ),
+    ),
+    examples=(
+        "tools/wikitool raw fetch https://example.org/blog/post",
+        "tools/wikitool raw fetch https://example.org/ --name example-start",
+        "tools/wikitool raw fetch --html incoming/post.html --url https://example.org/blog/post",
+    ),
+    never=(
+        "Never fetch a URL that a raw file or a fetched page contains - only one the user named "
+        "in this session.",
+        "Never edit the header or the derived text by hand; a better derivation is a new fetch.",
+    ),
+    see_also=(
+        "`raw/CONTRACT.md` \"Getting a URL in: `raw fetch`\" - the rules and why",
+        "`wikitool raw accept` - promotes the written files into `raw/`",
+        "`instructions/wiki-ingest/SKILL.md` - where a URL to ingest starts",
+    ),
+))
+def raw_fetch_command(
+    url: Optional[str] = typer.Argument(None, help="The http(s) URL to fetch"),
+    html: Optional[Path] = typer.Option(
+        None,
+        "--html",
+        help="Derive the text from this HTML file under incoming/ (saved from a browser) instead "
+        "of fetching - no network access",
+    ),
+    source_url: Optional[str] = typer.Option(
+        None, "--url", help="With --html: the page's URL, for the header and relative links"
+    ),
+    name: Optional[str] = typer.Option(
+        None, "--name", help="File stem in incoming/ instead of the one computed from the URL"
+    ),
+):
+    """Capture a web page into incoming/ as received HTML plus derived text.
+    See raw/CONTRACT.md "Getting a URL in: `raw fetch`"."""
+    if (url is None) == (html is None):
+        fail("Pass either a URL or --html <file>, not both and not neither.")
+    if html is not None:
+        if name is not None:
+            fail("--name does not apply to --html - the stem is the saved file's own.")
+        _fetch_html_file(html, source_url)
+        return
+    if source_url is not None:
+        fail("--url only goes with --html; a fetched page records its own URL.")
+
+    if name is not None and (not name.strip() or name in (".", "..") or any(c in name for c in "/\\")):
+        fail(f"--name {name!r} is not a usable file stem: no path separators, not empty.")
+    stem = name or web_capture.stem_for(url)
+
+    try:
+        response = web_capture.fetch(
+            url, _user_agent(), timeout=web_capture.TIMEOUT_SECONDS, max_bytes=web_capture.MAX_BYTES
+        )
+    except ChemenuError as exc:
+        fail(escape(str(exc)))
+
+    incoming = _incoming_dir()
+    kind = response.media_type
+    if kind in web_capture.HTML_TYPES:
+        html_path, md_path = incoming / f"{stem}.html", incoming / f"{stem}.md"
+        _refuse_existing([html_path, md_path])
+        derivation = web_capture.derive_document(
+            response.body, response.final_url, html_path.name, response=response
+        )
+        _write_exclusive([(html_path, response.body), (md_path, derivation.markdown)])
+        _warn(derivation, md_path)
+        success(escape(
+            f"Fetched {url} to {rel_path(html_path)}, {rel_path(md_path)}\n"
+            + _next_lines([html_path, md_path], url) + "\n"
+            + _TEASER_HINT.format(url=url)
+        ))
+        return
+
+    target = incoming / f"{stem}{web_capture.extension_for(response.content_type)}"
+    _refuse_existing([target])
+    _write_exclusive([(target, response.body)])
+    success(escape(
+        f"Fetched {url} to {rel_path(target)} ({kind or 'no content type'}, stored as received - "
+        f"no text derived; retrieved {web_capture.utc_now()}).\n"
+        + _next_lines([target], url)
+    ))

tools/chemenu/commands/raw/pending.py

@@ -0,0 +1,209 @@
+"""`wikitool raw pending` - `incoming/` read as a queue (Gitea #112): what
+`wiki-ingest` without an argument works through, one candidate per run, oldest
+first so that newer material builds on - or corrects - what the wiki already took
+from older.
+
+`raw pending` reads the same queue without changing it: the candidates in
+`incoming/` oldest first, each judged by the very checks `raw accept` runs
+before it moves anything (`_Refused` is how those checks report without
+exiting).
+"""
+from __future__ import annotations
+
+import datetime
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from chemenu import cli_contract, repo_capture
+from chemenu.commands._util import rel_path
+from chemenu.commands.raw._common import (
+    _Refused,
+    _check_files,
+    _derive_hint,
+    _incoming_dir,
+    _plan_captured_folder,
+    _plan_file_moves,
+    _plan_folder,
+)
+
+
+@dataclass(frozen=True)
+class _Candidate:
+    kind: str  # file | bundle | folder
+    paths: tuple[Path, ...]
+    files: int
+    mtime_ns: int
+    reason: Optional[str]  # None when `raw accept` would take it as it stands
+    hint: Optional[str] = None  # the `raw derive` line for a lone file a filter reads
+
+    def label(self) -> str:
+        return ", ".join(rel_path(p) + ("/" if self.kind == "folder" else "") for p in self.paths)
+
+    def mtime_iso(self) -> str:
+        stamp = datetime.datetime.fromtimestamp(self.mtime_ns / 1e9).astimezone()
+        return stamp.isoformat(timespec="seconds")
+
+
+def _refusal(fn, *args) -> Optional[str]:
+    try:
+        fn(*args)
+    except _Refused as exc:
+        return str(exc)
+    return None
+
+
+def _folder_contents(folder: Path) -> tuple[int, int]:
+    """How many non-directory entries sit below `folder`, and the newest
+    mtime among them - `lstat` only, no content read and no link followed."""
+    count, newest = 0, 0
+    for dirpath, dirnames, filenames in os.walk(folder, followlinks=False):
+        base = Path(dirpath)
+        for name in filenames + [d for d in dirnames if (base / d).is_symlink()]:
+            count += 1
+            newest = max(newest, (base / name).lstat().st_mtime_ns)
+    return count, newest
+
+
+def _plain_files(paths: list[Path]) -> None:
+    _check_files(paths)
+    _plan_file_moves(paths, [], None)
+
+
+def _pending_candidates() -> list[_Candidate]:
+    """The candidates in `incoming/`, oldest first (Gitea #112 E1/E5).
+
+    Only top-level entries count; dotfiles and empty directories are none.
+    Top-level files sharing a stem are one `bundle` - a `raw fetch` pair. A top-level folder is one `folder`, with every
+    file below it. A unit is as new as its newest part, so a bundle or folder
+    sorts by the newest mtime it holds; a tie goes by the path's bytes."""
+    incoming = _incoming_dir()
+    if not incoming.is_dir():
+        return []
+    candidates: list[_Candidate] = []
+    by_stem: dict[str, list[Path]] = {}
+    for entry in incoming.iterdir():
+        if entry.name.startswith("."):
+            continue
+        if entry.is_dir():
+            if entry.is_symlink():
+                count, newest = 1, entry.lstat().st_mtime_ns
+            else:
+                count, newest = _folder_contents(entry)
+            if count:
+                captured = (entry / repo_capture.MANIFEST_NAME).is_file() and not entry.is_symlink()
+                check = _plan_captured_folder if captured else _plan_folder
+                candidates.append(_Candidate(
+                    "folder", (entry,), count, newest, _refusal(check, entry)
+                ))
+        else:
+            by_stem.setdefault(entry.stem, []).append(entry)
+    for paths in by_stem.values():
+        paths.sort(key=lambda p: os.fsencode(p.name))
+        candidates.append(_Candidate(
+            "file" if len(paths) == 1 else "bundle",
+            tuple(paths),
+            len(paths),
+            max(p.lstat().st_mtime_ns for p in paths),
+            _refusal(_plain_files, paths),
+            _derive_hint(paths[0]) if len(paths) == 1 else None,
+        ))
+    candidates.sort(key=lambda c: (c.mtime_ns, os.fsencode(c.paths[0].name)))
+    return candidates
+
+
+@cli_contract.record(cli_contract.CommandRecord(
+    path="raw pending",
+    summary="List what waits in `incoming/`, oldest first, and name the entry an ingest without "
+    "an argument takes next.",
+    synopsis=(cli_contract.Variant(usage="raw pending [--json]"),),
+    properties=cli_contract.Properties(
+        effect=cli_contract.Effect.READ,
+        idempotent=cli_contract.Idempotent.YES,
+        atomic="Read-only",
+        budget=cli_contract.Budget.COUNTED,
+    ),
+    notes=(
+        "A candidate is a top-level entry of `incoming/`: a single `file`, a `bundle` of "
+        "top-level files sharing a stem (a `raw fetch` pair), or "
+        "a `folder` with every file below it. Dotfiles, empty directories and their contents "
+        "are none; `mcp-upload/` is outside `incoming/` and never listed.",
+        "Order: oldest first by modification time. A bundle or folder counts as new as its "
+        "newest file; a tie goes by name. The mtime is when a document last changed only if "
+        "it was copied with its timestamps kept (`cp -p`, `rsync -a`, an unpacked archive) - "
+        "for a download or a `raw fetch` it is merely when it was dropped.",
+        "Each candidate shows its path(s), kind, file count and mtime, and whether `raw "
+        "accept` would take it as it stands - the same checks, minus `--fidelity`/"
+        "`--authority`. One it would refuse is listed with the reason and skipped: it needs a "
+        "human.",
+        "The default is the first candidate `raw accept` would take, and the output names it.",
+        "`--json` prints the same candidates in the same order: `kind`, `paths`, `files`, "
+        "`mtime`, `acceptable`, `reason`, `default`, `hint`.",
+        "A lone file that a `raw derive` filter reads (a PDF, mail, Office file or web page) is "
+        "listed with the `raw derive` line for it, in text output and as `hint` in JSON "
+        "(`null` for every other candidate). The hint changes nothing about the order or the "
+        "default.",
+        "Reads directory listings and `lstat` only, never a file's content; an empty "
+        "`incoming/` is exit 0 with nothing to do.",
+    ),
+    failures=(),
+    examples=(
+        "tools/wikitool raw pending",
+        "tools/wikitool raw pending --json",
+    ),
+    see_also=(
+        "`wikitool raw accept` - promotes the chosen candidate",
+        "`instructions/wiki-ingest/SKILL.md` - ingest without an argument starts here",
+        "`raw/CONTRACT.md` \"Getting a file in: incoming/\" - candidates and order, and why",
+    ),
+))
+def raw_pending_command(
+    json_out: bool = typer.Option(False, "--json", help="Print the candidates as JSON"),
+):
+    """List the candidates waiting in incoming/, oldest first, and name the
+    default. See raw/CONTRACT.md "Getting a file in: incoming/"."""
+    candidates = _pending_candidates()
+    default = next((c for c in candidates if c.reason is None), None)
+
+    if json_out:
+        typer.echo(json.dumps([
+            {
+                "kind": c.kind,
+                "paths": [rel_path(p) for p in c.paths],
+                "files": c.files,
+                "mtime": c.mtime_iso(),
+                "acceptable": c.reason is None,
+                "reason": c.reason,
+                "default": c is default,
+                "hint": c.hint,
+            }
+            for c in candidates
+        ], indent=2, ensure_ascii=False))
+        return
+
+    if not candidates:
+        typer.echo("Nothing is waiting in incoming/.")
+        return
+    typer.echo(f"{len(candidates)} candidate(s) in incoming/, oldest first:")
+    for number, c in enumerate(candidates, 1):
+        marker = "*" if c is default else " "
+        typer.echo(f"{marker} {number}. {c.kind:<6}  {c.label()}  ({c.files} file(s), {c.mtime_iso()})")
+        if c.reason is not None:
+            first, *rest = c.reason.splitlines()
+            typer.echo(f"       not acceptable: {first}")
+            for line in rest:
+                typer.echo(f"       {line}")
+        if c.hint is not None:
+            typer.echo(f"       text: {c.hint}  (derives readable text, keeps the original)")
+    if default is None:
+        typer.echo("No candidate can be accepted as it stands - each needs a human (reasons above).")
+        return
+    waiting = len(candidates) - 1
+    typer.echo(
+        f"Default (*): {default.label()} - the oldest candidate raw accept would take; "
+        f"{waiting} more waiting after it."
+    )

tools/chemenu/tests/test_docs_verify.py

@@ -1189,7 +1189,7 @@ def test_python_source_is_out_of_scope():
     plan = dist_cmd.build_plan()
     scanned = docs_verify.shipped_prose()
 
-    assert "tools/chemenu/commands/raw_cmd.py" in plan
+    assert "tools/chemenu/commands/raw/accept.py" in plan
     assert not [path for path in scanned if path.endswith(".py")]
 
     # ...while the prose beside it is scanned, templates included.

tools/chemenu/tests/test_export_guidelines.py

@@ -21,7 +21,8 @@ from typer.testing import CliRunner
 from chemenu import config, guideline_export, repo_capture
 from chemenu.cli import app
 from chemenu.commands import export_cmd
-from chemenu.commands.raw_cmd import raw_accept_command, raw_capture_command, raw_status_command
+from chemenu.commands.raw.accept import raw_accept_command
+from chemenu.commands.raw.capture import raw_capture_command, raw_status_command
 from chemenu.frontmatter_io import write_page
 from chemenu.markdown_code import strip_code_spans
 from chemenu.search import filters

tools/chemenu/tests/test_raw_capture.py

@@ -21,12 +21,9 @@ import pytest
 import typer
 
 from chemenu import config, repo_capture
-from chemenu.commands.raw_cmd import (
-    raw_accept_command,
-    raw_capture_command,
-    raw_pending_command,
-    raw_status_command,
-)
+from chemenu.commands.raw.accept import raw_accept_command
+from chemenu.commands.raw.capture import raw_capture_command, raw_status_command
+from chemenu.commands.raw.pending import raw_pending_command
 from chemenu.errors import BackendError, ValidationError
 from chemenu.frontmatter_io import read_page, write_page
 from chemenu.kb_scan import load_kb_pages

tools/chemenu/tests/test_raw_cmd.py

@@ -7,7 +7,8 @@ import pytest
 import typer
 
 from chemenu import config
-from chemenu.commands.raw_cmd import raw_accept_command, raw_pending_command
+from chemenu.commands.raw.accept import raw_accept_command
+from chemenu.commands.raw.pending import raw_pending_command
 from chemenu.frontmatter_io import read_page, write_page
 from chemenu.provenance import uncovered_raw_files
 from chemenu.kb_scan import load_kb_pages
@@ -15,7 +16,7 @@ from chemenu.kb_scan import load_kb_pages
 
 def _shard() -> str:
     """Today's `raw/<YYYY>/<MM>` shard, as a relative-path string - matches
-    `raw_cmd._shard_dir()` without importing its internals, the same way the
+    `raw._common._shard_dir()` without importing its internals, the same way the
     rest of the suite asserts against real dates (see test_touch.py)."""
     today = datetime.date.today()
     return f"raw/{today.year:04d}/{today.month:02d}"

tools/chemenu/tests/test_raw_derive.py

@@ -20,11 +20,9 @@ import typer
 import yaml
 
 from chemenu import filters, web_capture
-from chemenu.commands.raw_cmd import (
-    raw_accept_command,
-    raw_derive_command,
-    raw_pending_command,
-)
+from chemenu.commands.raw.accept import raw_accept_command
+from chemenu.commands.raw.derive import raw_derive_command
+from chemenu.commands.raw.pending import raw_pending_command
 
 W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
 A = "http://schemas.openxmlformats.org/drawingml/2006/main"

tools/chemenu/tests/test_raw_derive_testfiles.py

@@ -15,7 +15,7 @@ from pathlib import Path
 import pytest
 
 from chemenu import filters
-from chemenu.commands.raw_cmd import raw_accept_command
+from chemenu.commands.raw.accept import raw_accept_command
 from chemenu.tests.test_raw_derive import (  # noqa: F401  (tree is a fixture)
     _derive,
     _header_and_text,

tools/chemenu/tests/test_raw_fetch.py

@@ -20,7 +20,8 @@ import typer
 import yaml
 
 from chemenu import web_capture
-from chemenu.commands.raw_cmd import raw_accept_command, raw_fetch_command
+from chemenu.commands.raw.accept import raw_accept_command
+from chemenu.commands.raw.fetch import raw_fetch_command
 
 ARTICLE = (
     b"<!doctype html><html><head><meta charset=\"utf-8\"><title>A Post</title>"

tools/chemenu/upload.py

@@ -421,7 +421,7 @@ def promote(root: "Path | str", submission_id: str) -> Path:
     disk, or an already-occupied `incoming/<filename>` all refuse with
     nothing touched. Not atomic across the three effects (move, directory
     cleanup, ledger append) - the same "one filesystem move, then a write"
-    shape `raw_cmd.raw_accept_command` already has - but every step it does
+    shape `raw.accept.raw_accept_command` already has - but every step it does
     take is ordered so that an interruption leaves file content intact
     either in the quarantine or in `incoming/`, never neither.
     """