torben/chemenu
1
1 Fork 0
Code Issues 35 Pull Requests Actions Packages Releases 37 Activity

feat: incoming/ as a queue - raw pending picks the next entry, raw accept takes a whole folder, a file in a subdirectory of incoming/ is refused (#112)
CI / verify (push) Successful in 5m24s
Details
CI / pwsh (push) Successful in 1m58s
Details
Release / release (push) Successful in 35s
Details

Browse Source 
Files changed:
- CHANGES.md
- README.md
- VERSION
- instructions/ingest-large-tree.md
- instructions/wiki-ingest/SKILL.md
- raw/CONTRACT.md
- tools/CONTRACT.md
- tools/chemenu/cli_contract.py
- tools/chemenu/commands/raw_cmd.py
- tools/chemenu/tests/test_raw_cmd.py
- tools/chemenu/tests/test_raw_fetch.py

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
This commit is contained in:
torbenandClaude Opus 5.5 committed 2026-10-03 09:49:20 +02:00
1 parent c4dcff76e6
commit 59c06e5ddc
11 files changed
+1009 -182
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/chemenu/commands/raw_cmd.py
+506 -111
View File
@@ -2,17 +2,22 @@
`raw/`, with the destination computed rather than chosen by hand (Gitea #58),
sharded by the accept date rather than by a hand-picked type (Gitea #67).
A human no longer classifies a file at all: `incoming/` is flat, and a
subdirectory dropped under it (an old `incoming/documents/` habit, a script
that still writes one) is accepted and ignored rather than inspected -
promoting `raw/` from a routing decision to an address computed purely from
*when* the file was accepted:
A human no longer classifies a file at all - `raw/` is an address computed
purely from *when* the file was accepted, and the kind of source comes from
its content (`source_type:` on the source page). So a subdirectory of
`incoming/` carries no type any more, and since Gitea #112 it is not tolerated
as one either: it **is** a source, accepted as a whole. A file argument must
sit directly in `incoming/`:
- **Single file, no bundle.** One file promoted alone lands as
`raw/<YYYY>/<MM>/<name>` - no directory of its own.
- **Bundle from the second file on.** Several files of one source promoted in
the same call land under `raw/<YYYY>/<MM>/<stem>/`, named after the first
file's stem.
- **A folder is one source** (Gitea #112). `raw accept incoming/<folder>`
moves every file below it to `raw/<YYYY>/<MM>/<folder>/` at the same
relative path, then removes the directories that are left empty. The folder
name is the bundle name.
- **Growing an existing single file into a bundle.** `--page` extends an
existing source page's `raw_files:`. If that raises the page from one file
to more than one, the file it already had is folded into a bundle at its
@@ -20,6 +25,11 @@ promoting `raw/` from a routing decision to an address computed purely from
today's shard, so a bundle never mixes an old capture date with today's
(Gitea #67 decision, "Datums-Shard" § "Bündelort").
`raw pending` reads the same queue without changing it: the candidates in
`incoming/` oldest first, each judged by the very checks `raw accept` runs
before it moves anything (`_Refused` is how those checks report without
exiting).
Existing files under `raw/` are never moved by this change (Gitea #67
"Altbestand bleibt stehen"): `raw/articles/`, `raw/documents/`, `raw/notes/`
and `raw/assets/` keep whatever they already held, and stay valid promotion
@@ -65,7 +75,10 @@ and names both routes rather than choosing one (Gitea #64 decision 2).
from __future__ import annotations
import datetime
import json
import os
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Optional
@@ -73,7 +86,7 @@ import typer
from rich.markup import escape
from chemenu import cli_contract, config, web_capture
from chemenu.commands._util import check_path_budget, fail, rel_path, success
from chemenu.commands._util import fail, path_budget_problem_for, rel_path, success
from chemenu.errors import ChemenuError
from chemenu.frontmatter_io import write_page
from chemenu.kb_scan import load_kb_pages
@@ -107,33 +120,51 @@ def _shard_dir(today: Optional[datetime.date] = None) -> Path:
return config.RAW_DIR / f"{d.year:04d}" / f"{d.month:02d}"
def _validate_under_incoming(path: Path, incoming: Path) -> None:
"""`path` must sit directly in `incoming/`, or exactly one level below it.
class _Refused(Exception):
"""A check that runs before anything moves has failed; the message is what
`fail()` prints. Raised rather than failing on the spot so `raw pending`
can ask the same checks of every candidate without exiting - one set of
checks, two callers (Gitea #112)."""
Unlike before #67, that one optional level carries no meaning any more -
it is accepted and ignored, kept only so an old `incoming/<type>/` habit
or script does not have to change to keep working (the MINOR condition
named in Gitea #67 "Versionsteil"). Nesting deeper than that is still
refused: it was never meaningful and silently accepting it would hide a
typo'd path.
def _refusals_fail(fn, *args):
try:
return fn(*args)
except _Refused as exc:
fail(escape(str(exc)))
def _check_directly_in_incoming(path: Path, incoming: Path) -> None:
"""`path` must sit directly in `incoming/`.
Up to Gitea #112 one subdirectory level was tolerated and ignored, so an
old `incoming/<type>/` habit kept working after #67 stopped reading the
type from it. A subdirectory is a source of its own now - accepted as a
whole - so a file inside one is refused with both ways out named.
"""
try:
rel = path.relative_to(incoming)
except ValueError:
fail(
raise _Refused(
f"{rel_path(path)} is not under incoming/ - `raw accept` and `raw fetch --html` "
"only take files from there. See raw/CONTRACT.md."
)
) from None
if len(rel.parts) < 1:
fail(f"incoming/{rel.as_posix()} names no file.")
if len(rel.parts) > 2:
fail(
f"incoming/{rel.as_posix()} is nested more than one level below incoming/ - place it "
"directly in incoming/, or in at most one subdirectory of it (the "
"subdirectory itself is ignored, see raw/CONTRACT.md)."
raise _Refused(f"incoming/{rel.as_posix()} names no file.")
if len(rel.parts) > 1:
top = rel.parts[0]
raise _Refused(
f"incoming/{rel.as_posix()} lies in a subdirectory of incoming/ - a file is taken "
"only from directly inside incoming/, and a subdirectory is a source of its own. "
f"Either accept the whole folder as one source (raw accept incoming/{top}), or move "
"the file up into incoming/ and accept it there. See raw/CONTRACT.md."
)
def _validate_under_incoming(path: Path, incoming: Path) -> None:
_refusals_fail(_check_directly_in_incoming, path, incoming)
_YEAR_DIR = re.compile(r"\d{4}")
@@ -185,6 +216,180 @@ def _stem_collision_message(claimed_name: str, holder: Path) -> str:
)
def _check_files(resolved: list[Path]) -> None:
incoming = _incoming_dir()
for path in resolved:
if not path.is_file():
raise _Refused(f"{rel_path(path)} does not exist or is not a file.")
_check_directly_in_incoming(path, incoming)
names = [path.name for path in resolved]
if len(names) != len(set(names)):
raise _Refused("Two files share a filename; rename one before promoting.")
def _check_target(dst: Path, remedy: str) -> None:
problem = path_budget_problem_for(dst)
if problem:
raise _Refused(f"Cannot write {problem}. {remedy}")
if dst.exists():
raise _Refused(f"Cannot promote: {rel_path(dst)} already exists.")
def _plan_file_moves(
resolved: list[Path], existing_raw_paths: list[Path], page: Optional[str]
) -> list[tuple[Path, Path]]:
"""Where each file goes - the bundle decision, the path budget, an existing
target and the name rule, all checked before anything moves. `page` and
`existing_raw_paths` are the `--page` case; `raw pending` asks with
neither."""
# A bundle directory forms once two or more files belong to the source
# (Gitea #58 decision 3): from the second file on, never before. Whenever
# --page targets an existing page it always has >=1 raw file already
# (types/source.md requires raw_files:), so bundling always applies there.
total = len(existing_raw_paths) + len(resolved)
bundle_dir: Optional[Path] = None
if len(existing_raw_paths) >= 2:
parents = {p.parent for p in existing_raw_paths}
if len(parents) != 1:
raise _Refused(
f"'{page}' raw_files: are not all in one directory - fix them by hand first "
"(see `sources coverage`)."
)
bundle_dir = parents.pop()
elif total >= 2:
if existing_raw_paths:
# Growing a bundle out of a single already-promoted file (Gitea
# #67 decision): the bundle forms at that file's own parent
# directory, never at today's shard - the file's capture date is
# whatever it always was, and a bundle mixing an old and a new
# shard would have no single correct address.
primary = existing_raw_paths[0]
bundle_dir = primary.parent / primary.stem
else:
primary = resolved[0]
bundle_dir = _shard_dir() / primary.stem
moves: list[tuple[Path, Path]] = []
for existing in existing_raw_paths:
if bundle_dir is not None and existing.parent != bundle_dir:
moves.append((existing, bundle_dir / existing.name))
for new_path in resolved:
dst = (bundle_dir / new_path.name) if bundle_dir is not None else (_shard_dir() / new_path.name)
moves.append((new_path, dst))
for _src, dst in moves:
_check_target(
dst,
"The name comes from the file in incoming/: rename it there to something shorter "
"and accept it again.",
)
# Stem uniqueness across raw/ (Gitea #64, widened by #67): the name this
# call is about to claim - the bundle's name, or the lone file's stem when
# no bundle forms - must not already belong to something this call does
# not itself own. "Owns" means: one of the page's already-registered raw
# files (the pitfall from the module docstring - a single file growing
# into a bundle of its own name momentarily still occupies that name), or,
# once a bundle already has >=2 registered files, the bundle directory
# itself.
claimed_name = bundle_dir.name if bundle_dir is not None else resolved[0].stem
occupied = _occupied_stems(config.RAW_DIR)
owned = set(existing_raw_paths)
if len(existing_raw_paths) >= 2:
owned.add(bundle_dir)
holder = occupied.get(claimed_name)
if holder is not None and holder not in owned:
raise _Refused(_stem_collision_message(claimed_name, holder))
return moves
def _walk_folder(folder: Path) -> tuple[list[Path], list[Path], list[str]]:
"""Every file below `folder`, every directory (`folder` included), and every
entry `raw accept` refuses to take - hidden or a symlink anywhere, or
neither a file nor a directory. Never descends into a refused entry, and
never follows a link."""
files: list[Path] = []
dirs: list[Path] = [folder]
refused: list[str] = []
for dirpath, dirnames, filenames in os.walk(folder, followlinks=False):
base = Path(dirpath)
descend = []
for name in sorted(dirnames + filenames):
entry = base / name
if name.startswith("."):
refused.append(f"{rel_path(entry)} (hidden)")
elif entry.is_symlink():
refused.append(f"{rel_path(entry)} (symlink)")
elif entry.is_dir():
dirs.append(entry)
descend.append(name)
elif entry.is_file():
files.append(entry)
else:
refused.append(f"{rel_path(entry)} (neither a file nor a directory)")
dirnames[:] = descend
return sorted(files), dirs, refused
def _folder_collision_message(name: str, holder: Path) -> str:
return (
f'{rel_path(holder)} already claims the name "{name}" under raw/.\n'
" Rename the folder in incoming/ (add a distinguishing suffix) and accept it again.\n"
" A folder has no --replaces: a later edition of one file in it is replaced file by file."
)
def _plan_folder(folder: Path) -> tuple[Path, list[tuple[Path, Path]], list[Path]]:
"""The bundle directory for `raw accept incoming/<folder>`, every move into
it, and the directories to remove afterwards, deepest first - or
`_Refused`, before anything moves.
The invariant the cleanup rests on (Gitea #112): after the moves no file
is left below `folder`, so removing the emptied directories can never
take one with it. That holds only because every entry the move would
skip - hidden, a symlink, a special file - is refused here, up front."""
incoming = _incoming_dir()
try:
rel = folder.relative_to(incoming)
except ValueError:
raise _Refused(
f"{rel_path(folder)} is not under incoming/ - `raw accept` only takes a folder from "
"there. See raw/CONTRACT.md."
) from None
if len(rel.parts) != 1:
raise _Refused(
f"incoming/{rel.as_posix()} is not directly in incoming/ - a folder is accepted only "
f"as a whole, from the top: raw accept incoming/{rel.parts[0]}"
)
if folder.name.startswith(".") or folder.is_symlink():
raise _Refused(f"{rel_path(folder)} is hidden or a symlink - raw accept does not take it.")
files, dirs, refused = _walk_folder(folder)
if refused:
listed = "\n".join(f" - {entry}" for entry in refused)
raise _Refused(
f"{rel_path(folder)}/ holds entries raw accept does not take - hidden entries, "
f"symlinks and special files are refused, so nothing is left behind:\n{listed}\n"
" Remove or replace them in incoming/, then accept the folder again."
)
if not files:
raise _Refused(f"{rel_path(folder)}/ holds no file - nothing to accept.")
holder = _occupied_stems(config.RAW_DIR).get(folder.name)
if holder is not None:
raise _Refused(_folder_collision_message(folder.name, holder))
bundle_dir = _shard_dir() / folder.name
moves = [(src, bundle_dir / src.relative_to(folder)) for src in files]
for _src, dst in moves:
_check_target(
dst,
"The path comes from the folder in incoming/: shorten the folder's name or the "
"names inside it, and accept it again.",
)
return bundle_dir, moves, sorted(dirs, key=lambda d: len(d.parts), reverse=True)
def _capture_choices() -> tuple[list[str], list[str]]:
"""Allowed `--fidelity`/`--authority` values, straight from the schema
(single source of truth) - `unknown` excluded, since it is backfill-only
@@ -312,9 +517,55 @@ def _replace(
success("Review them in this same run: the replacement and their update belong in one commit.")
def _accept_folder(folder: Path, fidelity: str, authority: str, dry_run: bool) -> None:
"""`raw accept incoming/<folder>` - one folder, one source (Gitea #112).
Not atomic: one move per file, then the emptied directories. A failure
part-way leaves a half-accepted folder, which is reported, not resumed -
the call is not repeated (tool error contract, case 4)."""
bundle_dir, moves, dirs = _refusals_fail(_plan_folder, folder)
if dry_run:
for src, dst in moves:
typer.echo(f"[dry-run] would move {rel_path(src)} -> {rel_path(dst)}")
typer.echo(f"[dry-run] would remove {rel_path(folder)}/ once it is empty")
typer.echo(f"[dry-run] would move {len(moves)} file(s). No files written.")
return
for src, dst in moves:
dst.parent.mkdir(parents=True, exist_ok=True)
src.rename(dst)
typer.echo(f" moved {rel_path(src)} -> {rel_path(dst)}")
# rmdir, never a recursive delete: it refuses a directory that still holds
# anything, so this step can only ever remove what the moves emptied.
left = []
for directory in dirs:
try:
directory.rmdir()
except OSError:
left.append(directory)
if left:
typer.echo(
f"WARN {', '.join(rel_path(d) for d in left)} could not be removed - something appeared "
"in it during the move. Every file of the folder was moved; look at what is left."
)
raw_files_arg = ",".join(rel_path(dst) for _src, dst in moves)
success(
f"Promoted {rel_path(folder)}/ ({len(moves)} file(s)) to {rel_path(bundle_dir)}/.\n"
" Next:\n"
" tools/wikitool new source --name \"<Title>\" \\\n"
f" --set raw_files={raw_files_arg} \\\n"
f" --set fidelity={fidelity} --set authority={authority} \\\n"
" --set source_type=<category>\n"
" Past the thresholds in instructions/ingest-large-tree.md, continue there instead:\n"
f" tools/wikitool work new --input {rel_path(bundle_dir)}"
)
@cli_contract.record(cli_contract.CommandRecord(
path="raw accept",
summary="Promote one or more files from `incoming/` into `raw/`.",
summary="Promote one or more files, or one folder, from `incoming/` into `raw/`.",
synopsis=(
cli_contract.Variant(
usage='raw accept <file> [<file> ...] --fidelity <v> --authority <v> '
@@ -322,6 +573,11 @@ def _replace(
notes="Promote one or more files from `incoming/` into today's `raw/<YYYY>/<MM>/` "
"shard",
),
cli_contract.Variant(
usage="raw accept incoming/<folder> --fidelity <v> --authority <v> [--dry-run]",
notes="Promote a whole folder as one source, its structure kept, into "
"`raw/<YYYY>/<MM>/<folder>/`",
),
cli_contract.Variant(
usage='raw accept <file> --replaces <raw-path> [--fidelity <v>] [--authority <v>] '
"[--dry-run]",
@@ -332,14 +588,23 @@ def _replace(
effect=cli_contract.Effect.WRITE,
idempotent=cli_contract.Idempotent.NO,
atomic="`raw accept`: No - one filesystem move per file, then (with `--page`) one page "
"write. `raw accept --replaces`: No - one `unlink()` + one `rename()`, plus (if "
"`--fidelity`/`--authority` was given) one page write",
"write. With a folder: No - one move per file, then one `rmdir` per emptied directory; "
"a half-accepted folder is not resumed. `raw accept --replaces`: No - one `unlink()` + "
"one `rename()`, plus (if `--fidelity`/`--authority` was given) one page write",
budget=cli_contract.Budget.COUNTED,
),
notes=(
"Promotes files from `incoming/` into `raw/<YYYY>/<MM>/`, computed from the accept "
"date rather than chosen by hand. A subdirectory under `incoming/` is tolerated and "
"ignored, not inspected - `raw/` does not address by type.",
"date rather than chosen by hand. A file argument must sit directly in `incoming/`; a "
"file inside a subdirectory is refused, since a subdirectory is a source of its own.",
"A folder argument (`incoming/<folder>`) is one source: every file below it moves to "
"`raw/<YYYY>/<MM>/<folder>/` at the same relative path, and the directories left empty "
"are removed - `incoming/<folder>` no longer exists afterwards. The folder name is the "
"bundle name; two `README.md` in different subfolders are no conflict.",
"A folder is accepted alone - no other argument, no `--page`, no `--replaces` - with "
"one `--fidelity`/`--authority` pair for all of it. It is refused, before anything "
"moves, if it is empty, or if a hidden entry (name starting with `.`), a symlink or a "
"special file sits anywhere below it; the refusal names each one.",
"One file promoted alone lands with no directory of its own; several files in one call "
"nest under `raw/<YYYY>/<MM>/<stem>/`, named after the first file's stem.",
"`--fidelity`/`--authority` are required on a plain accept (`types describe source` "
@@ -373,13 +638,23 @@ def _replace(
failures=(
cli_contract.Failure(
label="raw accept",
cause="A file does not exist, is not under `incoming/`, or is nested more than one "
"level below it; two files in one call share a filename; a target path already "
"exists; or a target path would be over the path budget (160 UTF-16 code units "
"below the instance root)",
reaction="Fix the named argument and retry once. For a path over the budget, "
"rename the file in `incoming/` to something shorter - the refusal comes before "
"anything moves, so `incoming/` and `raw/` are unchanged",
cause="A file does not exist or is not directly in `incoming/`; two files in one "
"call share a filename; a target path already exists; or a target path would be "
"over the path budget (160 UTF-16 code units below the instance root)",
reaction="Fix the named argument and retry once. A file inside a subdirectory of "
"`incoming/` is accepted with its whole folder (`raw accept incoming/<folder>`) or "
"moved up into `incoming/` first. For a path over the budget, rename the file in "
"`incoming/` to something shorter - the refusal comes before anything moves, so "
"`incoming/` and `raw/` are unchanged",
),
cli_contract.Failure(
label="raw accept incoming/<folder>",
cause="The folder is not directly in `incoming/`, is combined with another argument, "
"`--page` or `--replaces`, holds no file, or holds a hidden entry, a symlink or a "
"special file; a target path is over the budget; or the folder name is already "
"occupied under `raw/`",
reaction="Nothing moved. Fix what the message names and retry once; for an occupied "
"name, rename the folder in `incoming/` - there is no `--replaces` for a folder",
),
cli_contract.Failure(
label="raw accept",
@@ -411,9 +686,9 @@ def _replace(
),
cli_contract.Failure(
label="raw accept --replaces",
cause="The incoming file does not exist or is not under `incoming/` (or is nested "
"more than one level below it), its filename differs from the target's, or the "
"target does not lie under `raw/` or does not exist",
cause="The incoming file does not exist or is not directly in `incoming/`, its "
"filename differs from the target's, or the target does not lie under `raw/` or "
"does not exist",
reaction="Fix the named argument and retry once - every check runs before the "
"filesystem is touched, so both files are exactly as they were",
),
@@ -429,6 +704,7 @@ def _replace(
"--authority reporting",
'tools/wikitool raw accept incoming/part-2.md --fidelity verbatim --authority reporting '
'--page "Source - Docker Cheatsheet"',
"tools/wikitool raw accept incoming/projekt-x --fidelity verbatim --authority reporting",
"tools/wikitool raw accept incoming/cluster.md --replaces raw/documents/cluster.md",
),
never=(
@@ -439,6 +715,7 @@ def _replace(
),
see_also=(
"`raw/CONTRACT.md` \"Getting a file in: incoming/\" - the rules and why",
"`wikitool raw pending` - what is waiting in `incoming/`, and which entry is next",
"`wikitool types describe source` - the capture field values",
"`wikitool new source` - the source page for a promoted file",
),
@@ -447,7 +724,8 @@ def _replace(
def raw_accept_command(
files: list[Path] = typer.Argument(
...,
help="One or more files under incoming/, all belonging to the same source",
help="One or more files directly in incoming/, all belonging to the same source - or "
"one folder in incoming/, accepted whole as one source",
),
fidelity: Optional[str] = typer.Option(
None,
@@ -476,13 +754,19 @@ def raw_accept_command(
),
dry_run: bool = typer.Option(False, "--dry-run", help="List what would move without writing"),
):
"""Promote file(s) from incoming/ into raw/, computing the destination
(date shard, bundle or not, bundle name) instead of taking it as an
argument. See raw/CONTRACT.md "Getting a file in: incoming/"."""
"""Promote file(s) or one folder from incoming/ into raw/, computing the
destination (date shard, bundle or not, bundle name) instead of taking it
as an argument. See raw/CONTRACT.md "Getting a file in: incoming/"."""
if not files:
fail("Pass at least one file to promote.")
incoming = _incoming_dir()
resolved = [_resolve(f) for f in files]
folders = [p for p in resolved if p.is_dir()]
if folders and (len(files) != 1 or page is not None or replaces is not None):
fail(escape(
f"{rel_path(folders[0])} is a folder, and a folder is accepted alone: one source, "
"with no other argument, no --page and no --replaces. Accept it in a call of its own."
))
if replaces is not None:
_replace(files, replaces, page, fidelity, authority, dry_run)
@@ -498,16 +782,11 @@ def raw_accept_command(
_check_capture_value("fidelity", fidelity, fidelity_choices)
_check_capture_value("authority", authority, authority_choices)
resolved = [_resolve(f) for f in files]
if folders:
_accept_folder(folders[0], fidelity, authority, dry_run)
return
for path in resolved:
if not path.is_file():
fail(f"{rel_path(path)} does not exist or is not a file.")
_validate_under_incoming(path, incoming)
names = [path.name for path in resolved]
if len(names) != len(set(names)):
fail("Two files share a filename; rename one before promoting.")
_refusals_fail(_check_files, resolved)
pages = None
target_page = None
@@ -532,66 +811,7 @@ def raw_accept_command(
"before promoting more."
)
# A bundle directory forms once two or more files belong to the source
# (Gitea #58 decision 3): from the second file on, never before. Whenever
# --page targets an existing page it always has >=1 raw file already
# (types/source.md requires raw_files:), so bundling always applies there.
total = len(existing_raw_paths) + len(resolved)
bundle_dir: Optional[Path] = None
if len(existing_raw_paths) >= 2:
parents = {p.parent for p in existing_raw_paths}
if len(parents) != 1:
fail(
f"'{page}' raw_files: are not all in one directory - fix them by hand first "
"(see `sources coverage`)."
)
bundle_dir = parents.pop()
elif total >= 2:
if existing_raw_paths:
# Growing a bundle out of a single already-promoted file (Gitea
# #67 decision): the bundle forms at that file's own parent
# directory, never at today's shard - the file's capture date is
# whatever it always was, and a bundle mixing an old and a new
# shard would have no single correct address.
primary = existing_raw_paths[0]
bundle_dir = primary.parent / primary.stem
else:
primary = resolved[0]
bundle_dir = _shard_dir() / primary.stem
moves: list[tuple[Path, Path]] = []
for existing in existing_raw_paths:
if bundle_dir is not None and existing.parent != bundle_dir:
moves.append((existing, bundle_dir / existing.name))
for new_path in resolved:
dst = (bundle_dir / new_path.name) if bundle_dir is not None else (_shard_dir() / new_path.name)
moves.append((new_path, dst))
for _src, dst in moves:
check_path_budget(
dst,
"The name comes from the file in incoming/: rename it there to something shorter "
"and accept it again.",
)
if dst.exists():
fail(f"Cannot promote: {rel_path(dst)} already exists.")
# Stem uniqueness across raw/ (Gitea #64, widened by #67): the name this
# call is about to claim - the bundle's name, or the lone file's stem when
# no bundle forms - must not already belong to something this call does
# not itself own. "Owns" means: one of the page's already-registered raw
# files (the pitfall from the module docstring - a single file growing
# into a bundle of its own name momentarily still occupies that name), or,
# once a bundle already has >=2 registered files, the bundle directory
# itself.
claimed_name = bundle_dir.name if bundle_dir is not None else resolved[0].stem
occupied = _occupied_stems(config.RAW_DIR)
owned = set(existing_raw_paths)
if len(existing_raw_paths) >= 2:
owned.add(bundle_dir)
holder = occupied.get(claimed_name)
if holder is not None and holder not in owned:
fail(_stem_collision_message(claimed_name, holder))
moves = _refusals_fail(_plan_file_moves, resolved, existing_raw_paths, page)
moving_existing = [src for src, _dst in moves if src in existing_raw_paths]
if moving_existing:
@@ -663,6 +883,181 @@ def raw_accept_command(
)
# --- raw pending --------------------------------------------------------------
#
# `incoming/` read as a queue (Gitea #112): what `wiki-ingest` without an
# argument works through, one candidate per run, oldest first so that newer
# material builds on - or corrects - what the wiki already took from older.
@dataclass(frozen=True)
class _Candidate:
kind: str # file | bundle | folder
paths: tuple[Path, ...]
files: int
mtime_ns: int
reason: Optional[str] # None when `raw accept` would take it as it stands
def label(self) -> str:
return ", ".join(rel_path(p) + ("/" if self.kind == "folder" else "") for p in self.paths)
def mtime_iso(self) -> str:
stamp = datetime.datetime.fromtimestamp(self.mtime_ns / 1e9).astimezone()
return stamp.isoformat(timespec="seconds")
def _refusal(fn, *args) -> Optional[str]:
try:
fn(*args)
except _Refused as exc:
return str(exc)
return None
def _folder_contents(folder: Path) -> tuple[int, int]:
"""How many non-directory entries sit below `folder`, and the newest
mtime among them - `lstat` only, no content read and no link followed."""
count, newest = 0, 0
for dirpath, dirnames, filenames in os.walk(folder, followlinks=False):
base = Path(dirpath)
for name in filenames + [d for d in dirnames if (base / d).is_symlink()]:
count += 1
newest = max(newest, (base / name).lstat().st_mtime_ns)
return count, newest
def _plain_files(paths: list[Path]) -> None:
_check_files(paths)
_plan_file_moves(paths, [], None)
def _pending_candidates() -> list[_Candidate]:
"""The candidates in `incoming/`, oldest first (Gitea #112 E1/E5).
Only top-level entries count; dotfiles and empty directories are none.
Top-level files sharing a stem are one `bundle` - a `raw fetch` pair, a PDF
and its converted text. A top-level folder is one `folder`, with every
file below it. A unit is as new as its newest part, so a bundle or folder
sorts by the newest mtime it holds; a tie goes by the path's bytes."""
incoming = _incoming_dir()
if not incoming.is_dir():
return []
candidates: list[_Candidate] = []
by_stem: dict[str, list[Path]] = {}
for entry in incoming.iterdir():
if entry.name.startswith("."):
continue
if entry.is_dir():
if entry.is_symlink():
count, newest = 1, entry.lstat().st_mtime_ns
else:
count, newest = _folder_contents(entry)
if count:
candidates.append(_Candidate(
"folder", (entry,), count, newest, _refusal(_plan_folder, entry)
))
else:
by_stem.setdefault(entry.stem, []).append(entry)
for paths in by_stem.values():
paths.sort(key=lambda p: os.fsencode(p.name))
candidates.append(_Candidate(
"file" if len(paths) == 1 else "bundle",
tuple(paths),
len(paths),
max(p.lstat().st_mtime_ns for p in paths),
_refusal(_plain_files, paths),
))
candidates.sort(key=lambda c: (c.mtime_ns, os.fsencode(c.paths[0].name)))
return candidates
@cli_contract.record(cli_contract.CommandRecord(
path="raw pending",
summary="List what waits in `incoming/`, oldest first, and name the entry an ingest without "
"an argument takes next.",
synopsis=(cli_contract.Variant(usage="raw pending [--json]"),),
properties=cli_contract.Properties(
effect=cli_contract.Effect.READ,
idempotent=cli_contract.Idempotent.YES,
atomic="Read-only",
budget=cli_contract.Budget.COUNTED,
),
notes=(
"A candidate is a top-level entry of `incoming/`: a single `file`, a `bundle` of "
"top-level files sharing a stem (a `raw fetch` pair, a PDF and its converted text), or "
"a `folder` with every file below it. Dotfiles, empty directories and their contents "
"are none; `mcp-upload/` is outside `incoming/` and never listed.",
"Order: oldest first by modification time. A bundle or folder counts as new as its "
"newest file; a tie goes by name. The mtime is when a document last changed only if "
"it was copied with its timestamps kept (`cp -p`, `rsync -a`, an unpacked archive) - "
"for a download or a `raw fetch` it is merely when it was dropped.",
"Each candidate shows its path(s), kind, file count and mtime, and whether `raw "
"accept` would take it as it stands - the same checks, minus `--fidelity`/"
"`--authority`. One it would refuse is listed with the reason and skipped: it needs a "
"human.",
"The default is the first candidate `raw accept` would take, and the output names it.",
"`--json` prints the same candidates in the same order: `kind`, `paths`, `files`, "
"`mtime`, `acceptable`, `reason`, `default`.",
"Reads directory listings and `lstat` only, never a file's content; an empty "
"`incoming/` is exit 0 with nothing to do.",
),
failures=(),
examples=(
"tools/wikitool raw pending",
"tools/wikitool raw pending --json",
),
see_also=(
"`wikitool raw accept` - promotes the chosen candidate",
"`instructions/wiki-ingest/SKILL.md` - ingest without an argument starts here",
"`raw/CONTRACT.md` \"Getting a file in: incoming/\" - candidates and order, and why",
),
))
@app.command("pending")
def raw_pending_command(
json_out: bool = typer.Option(False, "--json", help="Print the candidates as JSON"),
):
"""List the candidates waiting in incoming/, oldest first, and name the
default. See raw/CONTRACT.md "Getting a file in: incoming/"."""
candidates = _pending_candidates()
default = next((c for c in candidates if c.reason is None), None)
if json_out:
typer.echo(json.dumps([
{
"kind": c.kind,
"paths": [rel_path(p) for p in c.paths],
"files": c.files,
"mtime": c.mtime_iso(),
"acceptable": c.reason is None,
"reason": c.reason,
"default": c is default,
}
for c in candidates
], indent=2, ensure_ascii=False))
return
if not candidates:
typer.echo("Nothing is waiting in incoming/.")
return
typer.echo(f"{len(candidates)} candidate(s) in incoming/, oldest first:")
for number, c in enumerate(candidates, 1):
marker = "*" if c is default else " "
typer.echo(f"{marker} {number}. {c.kind:<6} {c.label()} ({c.files} file(s), {c.mtime_iso()})")
if c.reason is not None:
first, *rest = c.reason.splitlines()
typer.echo(f" not acceptable: {first}")
for line in rest:
typer.echo(f" {line}")
if default is None:
typer.echo("No candidate can be accepted as it stands - each needs a human (reasons above).")
return
waiting = len(candidates) - 1
typer.echo(
f"Default (*): {default.label()} - the oldest candidate raw accept would take; "
f"{waiting} more waiting after it."
)
# --- raw fetch ----------------------------------------------------------------
#
# The sanctioned intake for a URL the user names. It ends in `incoming/`, never