diff --git a/CHANGES.md b/CHANGES.md index 452e99b..2b175db 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,13 +14,14 @@ place only: `instructions/dev/version-parts.md` § The candidate model. --- -## 9.0.1-beta.1 - 2026-10-10 - Ketten von Paketen: stack-dev schreibt ## Kette, stack-close nennt das nächste Glied +## 9.0.1-beta.2 - 2026-10-10 - raw_cmd.py in ein Paket zerlegt: ein Modul je Unterkommando, Ausgabe unverändert **Author:** Torben Nehmer **Low impact** - Ketten von Paketen: stack-dev schreibt ## Kette, stack-close nennt das nächste Glied +- raw_cmd.py in ein Paket zerlegt: ein Modul je Unterkommando, Ausgabe unverändert --- diff --git a/VERSION b/VERSION index b921001..3e54d33 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -9.0.1-beta.1 +9.0.1-beta.2 diff --git a/tools/README.md b/tools/README.md index d2cdd5b..055f56b 100644 --- a/tools/README.md +++ b/tools/README.md @@ -119,7 +119,7 @@ tools/ guideline_export.py `export guidelines`' core: select pages by `search` predicates, render them into one deterministic GUIDELINES.md, and recognise an export again (`EXPORT_MARKER`) - with no CLI attached search/ pluggable search backends, plus service.py - the search core tasks/ the task-tracker provider layer: protocol.py (TaskReader/TaskWriter), config.py (.wikitool-tasks.json), one module per adapter - no instruction ever learns which provider it is - commands/ one module per command or command group: the terminal adapters + commands/ one module per command or command group: the terminal adapters - except `raw/`, a package with one module per subcommand and the helpers several of them share in `_common.py` tests/ pytest suite - origin repository only, `dist export` leaves it out ``` diff --git a/tools/chemenu/cli.py b/tools/chemenu/cli.py index 71256df..fa553a9 100644 --- a/tools/chemenu/cli.py +++ b/tools/chemenu/cli.py @@ -42,7 +42,7 @@ try: new_page, page_ops, provenance_cmd, - raw_cmd, + raw, review_cmd, run_budget, search as search_module, @@ -167,7 +167,7 @@ app.add_typer(links_cmd.app, name="links") app.add_typer(index_build.app, name="index") app.add_typer(log_append.app, name="log") app.add_typer(provenance_cmd.app, name="sources") -app.add_typer(raw_cmd.app, name="raw") +app.add_typer(raw.app, name="raw") app.add_typer(upload_cmd.app, name="upload") app.add_typer(instructions_cmd.app, name="instructions") app.add_typer(run_budget.app, name="budget") diff --git a/tools/chemenu/commands/raw/__init__.py b/tools/chemenu/commands/raw/__init__.py new file mode 100644 index 0000000..3438707 --- /dev/null +++ b/tools/chemenu/commands/raw/__init__.py @@ -0,0 +1,19 @@ +"""`wikitool raw` - bring raw material into `incoming/`, and promote it from there +into `raw/`. + +One module per subcommand; what two of them share is in `_common`. The Typer +group is assembled here, and the modules are imported in the order the +subcommands are listed in `-h` and in the generated command reference. +""" +import typer + +from chemenu.commands.raw import accept, pending, fetch, derive, capture + +app = typer.Typer(help="Bring raw material into incoming/, and promote it from there into raw/.") + +app.command("accept")(accept.raw_accept_command) +app.command("pending")(pending.raw_pending_command) +app.command("fetch")(fetch.raw_fetch_command) +app.command("derive")(derive.raw_derive_command) +app.command("capture")(capture.raw_capture_command) +app.command("status")(capture.raw_status_command) diff --git a/tools/chemenu/commands/raw/_common.py b/tools/chemenu/commands/raw/_common.py new file mode 100644 index 0000000..06bcc97 --- /dev/null +++ b/tools/chemenu/commands/raw/_common.py @@ -0,0 +1,499 @@ +"""Helpers that more than one `wikitool raw` subcommand uses: the `incoming/` +directory and its refusals, the date shard under `raw/`, the stems already taken, +and the planning of file and folder moves. A helper only one subcommand uses +lives in that subcommand's module. +""" +from __future__ import annotations + +import datetime +import os +import re +from pathlib import Path +from typing import Optional + +from rich.markup import escape + +from chemenu import config, filters, guideline_export, repo_capture +from chemenu.commands._util import fail, path_budget_problem_for, rel_path +from chemenu.errors import ValidationError +from chemenu.provenance import citing_pages, source_pages_by_raw_file +from chemenu.type_resolver import resolver + + +# The type this command always promotes into eventually - hardcoded rather +# than derived from --page, because there is no page yet in the common case +# (see module docstring): the follow-up hint and the capture-field/enum +# validation below both need *a* type to read from, and `source` is the only +# one `raw accept` has ever scaffolded towards. +_SOURCE_TYPE_PATH = "types/source.md" + + +def _incoming_dir() -> Path: + return config.ROOT / "incoming" + + +def _resolve(raw: Path) -> Path: + return raw if raw.is_absolute() else config.ROOT / raw + + +def _shard_dir(today: Optional[datetime.date] = None) -> Path: + """`raw///`, computed from the accept date - never from a count, + so it can never rebalance and break a `[^cite-id]` anchor (Gitea #67). + """ + d = today or datetime.date.today() + return config.RAW_DIR / f"{d.year:04d}" / f"{d.month:02d}" + + +class _Refused(Exception): + """A check that runs before anything moves has failed; the message is what + `fail()` prints. Raised rather than failing on the spot so `raw pending` + can ask the same checks of every candidate without exiting - one set of + checks, two callers (Gitea #112).""" + + +def _refusals_fail(fn, *args): + try: + return fn(*args) + except _Refused as exc: + fail(escape(str(exc))) + + +def _check_directly_in_incoming(path: Path, incoming: Path) -> None: + """`path` must sit directly in `incoming/`. + + Up to Gitea #112 one subdirectory level was tolerated and ignored, so an + old `incoming//` habit kept working after #67 stopped reading the + type from it. A subdirectory is a source of its own now - accepted as a + whole - so a file inside one is refused with both ways out named. + """ + try: + rel = path.relative_to(incoming) + except ValueError: + raise _Refused( + f"{rel_path(path)} is not under incoming/ - `raw accept` and `raw fetch --html` " + "only take files from there. See raw/CONTRACT.md." + ) from None + if len(rel.parts) < 1: + raise _Refused(f"incoming/{rel.as_posix()} names no file.") + if len(rel.parts) > 1: + top = rel.parts[0] + raise _Refused( + f"incoming/{rel.as_posix()} lies in a subdirectory of incoming/ - a file is taken " + "only from directly inside incoming/, and a subdirectory is a source of its own. " + f"Either accept the whole folder as one source (raw accept incoming/{top}), or move " + "the file up into incoming/ and accept it there. See raw/CONTRACT.md." + ) + + +def _validate_under_incoming(path: Path, incoming: Path) -> None: + _refusals_fail(_check_directly_in_incoming, path, incoming) + + +_YEAR_DIR = re.compile(r"\d{4}") + + +def _occupied_stems(raw_dir: Path) -> dict[str, Path]: + """Every name occupied anywhere under `raw/`: file stems and bundle + directory names alike, at whichever level directly holds files. + + Distinguishes the pre-#67 flat layout (`raw//`) + from the #67 date shard (`raw///`) structurally, + by whether a top-level directory name is a 4-digit year - not from a + hardcoded list of legacy type names, so this keeps working unchanged if a + legacy directory is ever renamed by hand. + """ + occupied: dict[str, Path] = {} + if not raw_dir.is_dir(): + return occupied + for top in raw_dir.iterdir(): + if not top.is_dir(): + continue # raw/CONTRACT.md + if _YEAR_DIR.fullmatch(top.name): + for month in top.iterdir(): + if month.is_dir(): + occupied.update(_leaf_group_entries(month)) + else: + occupied.update(_leaf_group_entries(top)) + return occupied + + +def _leaf_group_entries(group_dir: Path) -> dict[str, Path]: + """The addressable entries directly inside one leaf-group directory + (a legacy type dir, or one `raw///`): a lone file occupies its + stem, a bundle directory occupies its own name.""" + return {entry.stem if entry.is_file() else entry.name: entry for entry in group_dir.iterdir()} + + +def _stem_collision_message(claimed_name: str, holder: Path) -> str: + example_target = holder + if holder.is_dir(): + children = sorted(holder.iterdir()) + example_target = children[0] if children else holder + return ( + f'{rel_path(holder)} already claims the stem "{claimed_name}" under raw/.\n' + " These are two different intents and only you can tell them apart:\n" + f" Same source, new edition -> tools/wikitool raw accept " + f"--replaces {rel_path(example_target)} \n" + " A second, separate source -> rename it in incoming/ (add a distinguishing " + "suffix) and accept it normally\n" + " raw accept does not guess which one this is." + ) + + +def _check_files(resolved: list[Path]) -> None: + incoming = _incoming_dir() + for path in resolved: + if not path.is_file(): + raise _Refused(f"{rel_path(path)} does not exist or is not a file.") + _check_directly_in_incoming(path, incoming) + if path.name == repo_capture.MANIFEST_NAME: + raise _Refused(_reserved_manifest_message(path)) + _check_no_export(resolved) + names = [path.name for path in resolved] + if len(names) != len(set(names)): + raise _Refused("Two files share a filename; rename one before promoting.") + + +def _check_no_export(paths: list[Path]) -> None: + """Refuse a file a wiki exported (`export guidelines`): it is generated + from `kb/`, so taking it into `raw/` would feed the wiki its own output + back as a source. `raw capture` already leaves such a file out; this + closes the way in through `incoming/`.""" + exports = [p for p in paths if guideline_export.file_is_export(p)] + if exports: + listed = "\n".join(f" - {rel_path(p)}" for p in exports) + raise _Refused( + "These files are a wiki's guideline export - their first line starts with " + "` markers and a WARN when it yields almost no text (a scan; there is " + "no OCR); an encrypted PDF is refused. A `.msg` is a container like an `.eml`; an " + "embedded message inside it is named, not read. The text of a mail (`.eml` and `.msg` " + "alike) is its plain text, unless that is under 10% of the length of the HTML text - a " + "\"plain text version not available\" placeholder - in which case it is the HTML text " + "and a WARN names both lengths. The other version of the text in a " + "`multipart/alternative` is not an attachment.", + "A workbook (`.xlsx`, `.xlsm`) gets its cells as files: beside its `.md` a folder " + "`.sheets/` (`derived.sheets/` for a single file) holds one `NN-.tsv` " + "per sheet - tab-separated per RFC 4180, UTF-8, row 1 of the file is row 1 of the sheet " + "- and, for a sheet with formulas, `NN-.formulas.tsv` listing cell, formula and " + "last stored value. The `.md` is the overview: one section per sheet with its range, " + "the file names and a preview. Formulas are not recalculated; a shared formula is " + "written out per cell, and one that cannot be is named `shared:` with a " + "WARN. The folder counts against the path budget and is accepted with the rest; an " + "existing one is never overwritten.", + "A ZIP and a mail are containers. A ZIP at the top of `incoming/` becomes the folder " + "`incoming//` holding its files under their own names, each with a `.md` " + "where a filter reads it; the ZIP is removed and the output names it with its sha256. A " + "mail becomes a folder with `original.eml`, `derived.md` and its attachments under their " + "own names; a ZIP attached to it is unpacked once into a folder of its own, and an " + "attached mail is quoted in `derived.md`, not unpacked. A ZIP inside a ZIP, an " + "absolute or `..` path, a symbolic link, an encrypted entry, more than 1000 entries, a " + "file over 25 MiB or more than 250 MiB in total refuse the whole call. Hidden entries " + "(`__MACOSX/`, `.DS_Store`, anything starting with a dot) are named, not written. A name " + "taken twice, or reserved (`original.*` and `derived.md` at the top of a mail's folder, " + "`_capture.json` anywhere), gets `-2`, `-3` before its extension, and the output lists " + "each rename.", + "A name already taken under `raw/` is a WARN, not a refusal: whether the folder is a " + "later edition of that source or a separate one is the human's call. A later edition of " + "a derived source is replaced file by file with `raw accept --replaces`; there is no " + "`--replaces-bundle` for it.", + "After a success the output prints the warnings the filters raised, at most three " + "`CHECK` lines per format that was derived in this run - what to look at before the " + "text is trusted - and the `raw accept` line for the result.", + "Never edit `derived.md` by hand: a better derivation is a new `raw derive` of the original.", + ), + failures=( + cli_contract.Failure( + cause="The path is not directly in `incoming/`, does not exist, or no filter reads " + "the file's extension", + reaction="Nothing was written. Fix the argument and retry once - a file in a " + "subfolder is moved up, or its folder is derived whole", + ), + cli_contract.Failure( + cause="The target folder, a `.md` or a workbook's `.sheets` folder already exists, or the path would be over the " + "path budget", + reaction="Nothing was written. Remove what is there or pass `--name `, " + "then retry once. An existing `.md` is never overwritten", + ), + cli_contract.Failure( + cause="The document cannot be read (damaged, encrypted), a container breaks one of its " + "rules (a ZIP inside a ZIP, an unsafe path, a limit) or the filter's package is not " + "installed", + reaction="Nothing was written. A missing package is reported with the preflight " + "hint - show it to the user and stop, never install it. Any other filter keeps " + "working", + ), + ), + examples=( + "tools/wikitool raw derive incoming/report.docx", + "tools/wikitool raw derive incoming/Angebot-XYZ.xlsx --name angebot-xyz", + "tools/wikitool raw derive incoming/meeting-files", + "tools/wikitool raw derive incoming/scans.zip", + ), + never=( + "Never edit `derived.md` or its header by hand.", + "Never derive a file the user did not put into `incoming/` for this purpose.", + ), + see_also=( + "`raw/CONTRACT.md` \"Getting a document in: `raw derive`\" - the rules and why", + "`wikitool raw accept` - promotes the folder into `raw/`", + "`instructions/wiki-ingest/SKILL.md` - where a PDF, mail or Office file to ingest starts", + ), +)) +def raw_derive_command( + path: Path = typer.Argument(..., help="A file or a folder directly in incoming/"), + name: Annotated[Optional[str], typer.Option( + "--name", help="Folder stem for a single file instead of the file's own stem" + )] = None, +): + """Derive reading text from a document in incoming/ and keep the original. + See raw/CONTRACT.md "Getting a document in: `raw derive`".""" + resolved = _resolve(path) + if resolved.is_dir(): + if name is not None: + fail("--name applies to a single file; a folder keeps its own name.") + derived, untouched = _refusals_fail(_plan_derive_folder, resolved) + _write_exclusive([write for d in derived for write in [(d.target, d.markdown), *d.extras]]) + lines = [f"Derived {len(derived)} file(s) in {rel_path(resolved)}/:"] + for d in derived: + more = f" + {len(d.extras)} file(s) in {rel_path(d.extras[0][0].parent)}/" if d.extras else "" + lines.append(f" {rel_path(d.source)} -> {rel_path(d.target)} ({d.filter.name}){more}") + if untouched: + lines.append( + " No filter for, left as they are: " + ", ".join(rel_path(p) for p in untouched) + ) + success(escape("\n".join(lines))) + warning = _collision_warning(resolved.name) + if warning: + typer.echo("WARN " + warning) + _echo_notes([d.derivation for d in derived]) + typer.echo(_derive_next(resolved)) + return + + plan = _refusals_fail(_plan_derive_file, resolved, name) + _write_folder(resolved, plan) + container = plan.container + if plan.original is None: + success(escape( + f"Unpacked {resolved.name} into {rel_path(plan.folder)}/ ({plan.candidate.name}); " + f"{resolved.name} is removed." + )) + elif container is not None and container.files: + success(escape( + f"Derived {rel_path(plan.derived)} from {resolved.name} ({plan.candidate.name}) with " + f"{len(container.files)} file(s) from its attachments; the original is now " + f"{rel_path(plan.original)}." + )) + else: + success(escape( + f"Derived {rel_path(plan.derived)} from {resolved.name} ({plan.candidate.name}); the " + f"original is now {rel_path(plan.original)}." + )) + if plan.extras: + lines = f" + {len(plan.extras)} file(s) beside it in {rel_path(plan.folder)}/{plan.extras[0][0].rsplit('/', 1)[0]}/" + typer.echo(escape(lines)) + if container is not None: + report = _container_report(container) + if report: + typer.echo("\n".join(report)) + warning = _collision_warning(plan.stem) + if warning: + typer.echo("WARN " + warning) + _echo_notes(plan.derivations, container) + typer.echo(_derive_next(plan.folder)) diff --git a/tools/chemenu/commands/raw/fetch.py b/tools/chemenu/commands/raw/fetch.py new file mode 100644 index 0000000..a74ef62 --- /dev/null +++ b/tools/chemenu/commands/raw/fetch.py @@ -0,0 +1,258 @@ +"""`wikitool raw fetch` - the sanctioned intake for a URL the user names. It ends +in `incoming/`, never in `raw/`: promoting stays `raw accept`'s job, so +`wiki-ingest` keeps its order (the commitment question comes before the promotion) +and the capture fields are asked exactly once, there. The fetch and derivation +themselves live in `chemenu.web_capture`; this adapter decides file names and what +the caller is told. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Optional + +import typer +from rich.markup import escape + +from chemenu import cli_contract, web_capture +from chemenu.commands._util import fail, rel_path, success +from chemenu.errors import ChemenuError +from chemenu.version import VersionError, read_version +from chemenu.commands.raw._common import ( + _incoming_dir, + _resolve, + _validate_under_incoming, + _write_exclusive, +) + + +_TEASER_HINT = ( + " If the text stops at a teaser (paywall, login, script-rendered page): save the page from a\n" + " logged-in browser to incoming/ and run tools/wikitool raw fetch --html " + "incoming/.html --url {url}" +) + + +def _user_agent() -> str: + try: + version = str(read_version()) + except VersionError: + version = "unknown" + return f"chemenu-wikitool/{version} (raw fetch)" + + +def _next_lines(files: list[Path], source_url: str) -> str: + paths = " ".join(rel_path(f) for f in files) + return ( + " Next (after the commitment question in wiki-ingest):\n" + f" tools/wikitool raw accept --fidelity published --authority {paths}\n" + f" and on the source page: --set source_url={source_url}" + ) + + +def _refuse_existing(targets: list[Path]) -> None: + existing = [t for t in targets if t.exists()] + if existing: + fail(escape( + f"{', '.join(rel_path(t) for t in existing)} already exists in incoming/ - raw fetch " + "never overwrites. Accept or remove the file that is there, or pass --name ." + )) + + +def _warn(derivation: web_capture.Derivation, md_path: Path) -> None: + if derivation.decoded.replaced: + typer.echo( + f"WARN {derivation.decoded.replaced} undecodable byte(s) under " + f"{derivation.decoded.charset} were replaced with U+FFFD in {rel_path(md_path)} - " + "the .html beside it keeps the original bytes." + ) + if derivation.text_chars < web_capture.SHORT_TEXT_CHARS: + typer.echo( + f"WARN The derived text is only {derivation.text_chars} characters - likely a " + "script-rendered page, a login wall or an empty response. Read " + f"{rel_path(md_path)} before accepting it." + ) + + +def _fetch_html_file(html: Path, source_url: Optional[str]) -> None: + if source_url is None: + fail("--html needs --url : it goes into the header, resolves relative links, and " + "becomes source_url on the source page.") + try: + web_capture.check_url(source_url) + except ChemenuError as exc: + fail(escape(str(exc))) + path = _resolve(html) + _validate_under_incoming(path, _incoming_dir()) + if not path.is_file(): + fail(f"{rel_path(path)} does not exist or is not a file.") + md_path = path.with_suffix(".md") + if md_path == path: + fail(f"{rel_path(path)} is already a .md file - --html takes the saved HTML page.") + _refuse_existing([md_path]) + + derivation = web_capture.derive_document(path.read_bytes(), source_url, path.name, url=source_url) + _write_exclusive([(md_path, derivation.markdown)]) + _warn(derivation, md_path) + success(escape( + f"Derived {rel_path(md_path)} from {rel_path(path)} (no network access).\n" + + _next_lines([path, md_path], source_url) + )) + + +@cli_contract.record(cli_contract.CommandRecord( + path="raw fetch", + summary="Capture a web page the user names into `incoming/`: the HTML as received plus a " + "derived text, for `raw accept` to promote.", + synopsis=( + cli_contract.Variant( + usage="raw fetch [--name ]", + notes="Fetch the page and write `incoming/.html` and `incoming/.md`", + ), + cli_contract.Variant( + usage="raw fetch --html incoming/.html --url ", + notes="Derive the `.md` from a page a human saved from their browser - no network " + "access", + ), + ), + properties=cli_contract.Properties( + effect=cli_contract.Effect.WRITE, + idempotent=cli_contract.Idempotent.NO, + atomic="Yes for what it leaves behind - one or two new files in `incoming/`, created " + "exclusively; a failure on the second removes the first", + budget=cli_contract.Budget.COUNTED, + network=cli_contract.Network.YES, + ), + notes=( + "Writes into `incoming/` only, never into `raw/`: `raw accept` promotes both files " + "afterwards, in one call, as one bundle under `raw////`.", + "The `.html` holds the response body byte for byte. The `.md` starts with a fixed " + "header block (`fetched_by`, `url`, `final_url`, `retrieved`, `http_status`, " + "`content_type`, `charset`, `title`, `derived_from`) followed by the derived text.", + "Character set, in this order: byte-order mark, the HTTP `Content-Type` charset, a " + "`` declaration in the first 4 KiB, UTF-8. Undecodable bytes are replaced with " + "U+FFFD and reported; the header names the charset and where it came from.", + "Derived text: the content root is `
`, else the single `
`, else " + "``; `script`, `style`, `noscript`, `nav`, `header`, `footer`, `aside`, `form`, " + "`template` and `svg` are dropped below it. Headings, lists, code, tables and links " + "(made absolute) become Markdown. The same bytes always give the same text.", + "The stem is the URL's last path segment without its extension, else its host name - " + "ASCII, lowercase, `-`-separated, at most 60 characters; `--name` overrides it.", + "A `text/plain` or `text/markdown` response is stored as received as `.txt`/`.md`; any " + "other non-HTML response (PDF, image, ...) as received under the extension of its " + "content type. Neither gets a derived file or a header.", + "Only `http`/`https`, on every redirect hop too. 30 s for the whole transfer, 25 MiB at " + "most. No cookies, no JavaScript: a derived text under 200 characters is written, with " + "a warning to read it before accepting.", + "`--html` derives the `.md` beside a page saved to `incoming/` from a logged-in browser " + "- the way past a paywall or a script-rendered page. Its header carries `derived` (when " + "the text was derived) instead of `retrieved`, `final_url`, `http_status` and " + "`content_type`, and the `.html` is left untouched.", + "Success prints the `raw accept` line for the written files and the `source_url` for the " + "source page.", + ), + failures=( + cli_contract.Failure( + cause="The URL is not `http`/`https` (also after a redirect), or neither or both of " + "`` and `--html` were given", + reaction="Fix the call and retry once. A local file is dropped into `incoming/` by " + "hand, not fetched", + ), + cli_contract.Failure( + cause="A target file already exists in `incoming/`", + reaction="Nothing was written or overwritten. Accept or remove what is there, or " + "pass `--name `, then retry once", + ), + cli_contract.Failure( + cause="The server answered with an HTTP error, could not be reached, took longer " + "than 30 s, or sent more than 25 MiB", + reaction="Nothing was written. An HTTP 4xx is not fixed by retrying - check the URL " + "with the user; for a paywall or login, save the page in a browser and use " + "`--html`. An unreachable host or a timeout may be retried once", + ), + cli_contract.Failure( + label="raw fetch --html", + cause="`--url` is missing, the file is not directly in `incoming/` or does not " + "exist, or `incoming/.md` already exists", + reaction="Fix the named argument and retry once - nothing was written. A saved page " + "inside a subdirectory of `incoming/` is moved up into `incoming/` first", + ), + ), + examples=( + "tools/wikitool raw fetch https://example.org/blog/post", + "tools/wikitool raw fetch https://example.org/ --name example-start", + "tools/wikitool raw fetch --html incoming/post.html --url https://example.org/blog/post", + ), + never=( + "Never fetch a URL that a raw file or a fetched page contains - only one the user named " + "in this session.", + "Never edit the header or the derived text by hand; a better derivation is a new fetch.", + ), + see_also=( + "`raw/CONTRACT.md` \"Getting a URL in: `raw fetch`\" - the rules and why", + "`wikitool raw accept` - promotes the written files into `raw/`", + "`instructions/wiki-ingest/SKILL.md` - where a URL to ingest starts", + ), +)) +def raw_fetch_command( + url: Optional[str] = typer.Argument(None, help="The http(s) URL to fetch"), + html: Optional[Path] = typer.Option( + None, + "--html", + help="Derive the text from this HTML file under incoming/ (saved from a browser) instead " + "of fetching - no network access", + ), + source_url: Optional[str] = typer.Option( + None, "--url", help="With --html: the page's URL, for the header and relative links" + ), + name: Optional[str] = typer.Option( + None, "--name", help="File stem in incoming/ instead of the one computed from the URL" + ), +): + """Capture a web page into incoming/ as received HTML plus derived text. + See raw/CONTRACT.md "Getting a URL in: `raw fetch`".""" + if (url is None) == (html is None): + fail("Pass either a URL or --html , not both and not neither.") + if html is not None: + if name is not None: + fail("--name does not apply to --html - the stem is the saved file's own.") + _fetch_html_file(html, source_url) + return + if source_url is not None: + fail("--url only goes with --html; a fetched page records its own URL.") + + if name is not None and (not name.strip() or name in (".", "..") or any(c in name for c in "/\\")): + fail(f"--name {name!r} is not a usable file stem: no path separators, not empty.") + stem = name or web_capture.stem_for(url) + + try: + response = web_capture.fetch( + url, _user_agent(), timeout=web_capture.TIMEOUT_SECONDS, max_bytes=web_capture.MAX_BYTES + ) + except ChemenuError as exc: + fail(escape(str(exc))) + + incoming = _incoming_dir() + kind = response.media_type + if kind in web_capture.HTML_TYPES: + html_path, md_path = incoming / f"{stem}.html", incoming / f"{stem}.md" + _refuse_existing([html_path, md_path]) + derivation = web_capture.derive_document( + response.body, response.final_url, html_path.name, response=response + ) + _write_exclusive([(html_path, response.body), (md_path, derivation.markdown)]) + _warn(derivation, md_path) + success(escape( + f"Fetched {url} to {rel_path(html_path)}, {rel_path(md_path)}\n" + + _next_lines([html_path, md_path], url) + "\n" + + _TEASER_HINT.format(url=url) + )) + return + + target = incoming / f"{stem}{web_capture.extension_for(response.content_type)}" + _refuse_existing([target]) + _write_exclusive([(target, response.body)]) + success(escape( + f"Fetched {url} to {rel_path(target)} ({kind or 'no content type'}, stored as received - " + f"no text derived; retrieved {web_capture.utc_now()}).\n" + + _next_lines([target], url) + )) diff --git a/tools/chemenu/commands/raw/pending.py b/tools/chemenu/commands/raw/pending.py new file mode 100644 index 0000000..bd636e5 --- /dev/null +++ b/tools/chemenu/commands/raw/pending.py @@ -0,0 +1,209 @@ +"""`wikitool raw pending` - `incoming/` read as a queue (Gitea #112): what +`wiki-ingest` without an argument works through, one candidate per run, oldest +first so that newer material builds on - or corrects - what the wiki already took +from older. + +`raw pending` reads the same queue without changing it: the candidates in +`incoming/` oldest first, each judged by the very checks `raw accept` runs +before it moves anything (`_Refused` is how those checks report without +exiting). +""" +from __future__ import annotations + +import datetime +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import typer + +from chemenu import cli_contract, repo_capture +from chemenu.commands._util import rel_path +from chemenu.commands.raw._common import ( + _Refused, + _check_files, + _derive_hint, + _incoming_dir, + _plan_captured_folder, + _plan_file_moves, + _plan_folder, +) + + +@dataclass(frozen=True) +class _Candidate: + kind: str # file | bundle | folder + paths: tuple[Path, ...] + files: int + mtime_ns: int + reason: Optional[str] # None when `raw accept` would take it as it stands + hint: Optional[str] = None # the `raw derive` line for a lone file a filter reads + + def label(self) -> str: + return ", ".join(rel_path(p) + ("/" if self.kind == "folder" else "") for p in self.paths) + + def mtime_iso(self) -> str: + stamp = datetime.datetime.fromtimestamp(self.mtime_ns / 1e9).astimezone() + return stamp.isoformat(timespec="seconds") + + +def _refusal(fn, *args) -> Optional[str]: + try: + fn(*args) + except _Refused as exc: + return str(exc) + return None + + +def _folder_contents(folder: Path) -> tuple[int, int]: + """How many non-directory entries sit below `folder`, and the newest + mtime among them - `lstat` only, no content read and no link followed.""" + count, newest = 0, 0 + for dirpath, dirnames, filenames in os.walk(folder, followlinks=False): + base = Path(dirpath) + for name in filenames + [d for d in dirnames if (base / d).is_symlink()]: + count += 1 + newest = max(newest, (base / name).lstat().st_mtime_ns) + return count, newest + + +def _plain_files(paths: list[Path]) -> None: + _check_files(paths) + _plan_file_moves(paths, [], None) + + +def _pending_candidates() -> list[_Candidate]: + """The candidates in `incoming/`, oldest first (Gitea #112 E1/E5). + + Only top-level entries count; dotfiles and empty directories are none. + Top-level files sharing a stem are one `bundle` - a `raw fetch` pair. A top-level folder is one `folder`, with every + file below it. A unit is as new as its newest part, so a bundle or folder + sorts by the newest mtime it holds; a tie goes by the path's bytes.""" + incoming = _incoming_dir() + if not incoming.is_dir(): + return [] + candidates: list[_Candidate] = [] + by_stem: dict[str, list[Path]] = {} + for entry in incoming.iterdir(): + if entry.name.startswith("."): + continue + if entry.is_dir(): + if entry.is_symlink(): + count, newest = 1, entry.lstat().st_mtime_ns + else: + count, newest = _folder_contents(entry) + if count: + captured = (entry / repo_capture.MANIFEST_NAME).is_file() and not entry.is_symlink() + check = _plan_captured_folder if captured else _plan_folder + candidates.append(_Candidate( + "folder", (entry,), count, newest, _refusal(check, entry) + )) + else: + by_stem.setdefault(entry.stem, []).append(entry) + for paths in by_stem.values(): + paths.sort(key=lambda p: os.fsencode(p.name)) + candidates.append(_Candidate( + "file" if len(paths) == 1 else "bundle", + tuple(paths), + len(paths), + max(p.lstat().st_mtime_ns for p in paths), + _refusal(_plain_files, paths), + _derive_hint(paths[0]) if len(paths) == 1 else None, + )) + candidates.sort(key=lambda c: (c.mtime_ns, os.fsencode(c.paths[0].name))) + return candidates + + +@cli_contract.record(cli_contract.CommandRecord( + path="raw pending", + summary="List what waits in `incoming/`, oldest first, and name the entry an ingest without " + "an argument takes next.", + synopsis=(cli_contract.Variant(usage="raw pending [--json]"),), + properties=cli_contract.Properties( + effect=cli_contract.Effect.READ, + idempotent=cli_contract.Idempotent.YES, + atomic="Read-only", + budget=cli_contract.Budget.COUNTED, + ), + notes=( + "A candidate is a top-level entry of `incoming/`: a single `file`, a `bundle` of " + "top-level files sharing a stem (a `raw fetch` pair), or " + "a `folder` with every file below it. Dotfiles, empty directories and their contents " + "are none; `mcp-upload/` is outside `incoming/` and never listed.", + "Order: oldest first by modification time. A bundle or folder counts as new as its " + "newest file; a tie goes by name. The mtime is when a document last changed only if " + "it was copied with its timestamps kept (`cp -p`, `rsync -a`, an unpacked archive) - " + "for a download or a `raw fetch` it is merely when it was dropped.", + "Each candidate shows its path(s), kind, file count and mtime, and whether `raw " + "accept` would take it as it stands - the same checks, minus `--fidelity`/" + "`--authority`. One it would refuse is listed with the reason and skipped: it needs a " + "human.", + "The default is the first candidate `raw accept` would take, and the output names it.", + "`--json` prints the same candidates in the same order: `kind`, `paths`, `files`, " + "`mtime`, `acceptable`, `reason`, `default`, `hint`.", + "A lone file that a `raw derive` filter reads (a PDF, mail, Office file or web page) is " + "listed with the `raw derive` line for it, in text output and as `hint` in JSON " + "(`null` for every other candidate). The hint changes nothing about the order or the " + "default.", + "Reads directory listings and `lstat` only, never a file's content; an empty " + "`incoming/` is exit 0 with nothing to do.", + ), + failures=(), + examples=( + "tools/wikitool raw pending", + "tools/wikitool raw pending --json", + ), + see_also=( + "`wikitool raw accept` - promotes the chosen candidate", + "`instructions/wiki-ingest/SKILL.md` - ingest without an argument starts here", + "`raw/CONTRACT.md` \"Getting a file in: incoming/\" - candidates and order, and why", + ), +)) +def raw_pending_command( + json_out: bool = typer.Option(False, "--json", help="Print the candidates as JSON"), +): + """List the candidates waiting in incoming/, oldest first, and name the + default. See raw/CONTRACT.md "Getting a file in: incoming/".""" + candidates = _pending_candidates() + default = next((c for c in candidates if c.reason is None), None) + + if json_out: + typer.echo(json.dumps([ + { + "kind": c.kind, + "paths": [rel_path(p) for p in c.paths], + "files": c.files, + "mtime": c.mtime_iso(), + "acceptable": c.reason is None, + "reason": c.reason, + "default": c is default, + "hint": c.hint, + } + for c in candidates + ], indent=2, ensure_ascii=False)) + return + + if not candidates: + typer.echo("Nothing is waiting in incoming/.") + return + typer.echo(f"{len(candidates)} candidate(s) in incoming/, oldest first:") + for number, c in enumerate(candidates, 1): + marker = "*" if c is default else " " + typer.echo(f"{marker} {number}. {c.kind:<6} {c.label()} ({c.files} file(s), {c.mtime_iso()})") + if c.reason is not None: + first, *rest = c.reason.splitlines() + typer.echo(f" not acceptable: {first}") + for line in rest: + typer.echo(f" {line}") + if c.hint is not None: + typer.echo(f" text: {c.hint} (derives readable text, keeps the original)") + if default is None: + typer.echo("No candidate can be accepted as it stands - each needs a human (reasons above).") + return + waiting = len(candidates) - 1 + typer.echo( + f"Default (*): {default.label()} - the oldest candidate raw accept would take; " + f"{waiting} more waiting after it." + ) diff --git a/tools/chemenu/commands/raw_cmd.py b/tools/chemenu/commands/raw_cmd.py deleted file mode 100644 index ecd787a..0000000 --- a/tools/chemenu/commands/raw_cmd.py +++ /dev/null @@ -1,2734 +0,0 @@ -"""`wikitool raw accept` - promote one or more files from `incoming/` into -`raw/`, with the destination computed rather than chosen by hand (Gitea #58), -sharded by the accept date rather than by a hand-picked type (Gitea #67). - -A human no longer classifies a file at all - `raw/` is an address computed -purely from *when* the file was accepted, and the kind of source comes from -its content (`source_type:` on the source page). So a subdirectory of -`incoming/` carries no type any more, and since Gitea #112 it is not tolerated -as one either: it **is** a source, accepted as a whole. A file argument must -sit directly in `incoming/`: - -- **Single file, no bundle.** One file promoted alone lands as - `raw///` - no directory of its own. -- **Bundle from the second file on.** Several files of one source promoted in - the same call land under `raw////`, named after the first - file's stem. -- **A folder is one source** (Gitea #112). `raw accept incoming/` - moves every file below it to `raw////` at the same - relative path, then removes the directories that are left empty. The folder - name is the bundle name. -- **Growing an existing single file into a bundle.** `--page` extends an - existing source page's `raw_files:`. If that raises the page from one file - to more than one, the file it already had is folded into a bundle at its - own parent directory - `raw///` - not at - today's shard, so a bundle never mixes an old capture date with today's - (Gitea #67 decision, "Datums-Shard" § "Bündelort"). - -`raw derive` is the way in for a document that is not text (PDF, mail, Office -file): it turns one file in `incoming/` into a folder `/` holding the -untouched `original.` and a `derived.md` made by the registered filter -(`chemenu.filters`), so `raw accept` takes the folder as one source. Everything -is checked before the first write; the original moves in last. - -`raw pending` reads the same queue without changing it: the candidates in -`incoming/` oldest first, each judged by the very checks `raw accept` runs -before it moves anything (`_Refused` is how those checks report without -exiting). - -Existing files under `raw/` are never moved by this change (Gitea #67 -"Altbestand bleibt stehen"): `raw/articles/`, `raw/documents/`, `raw/notes/` -and `raw/assets/` keep whatever they already held, and stay valid promotion -targets for `--replaces`. - -**Every promotion now also carries `--fidelity` and `--authority`** (Gitea -#67): how faithfully the material was captured, and what it is entitled to -claim about its subject. Neither has a default and neither may be `unknown` -here - that value is backfill-only, written only by `wikitool touch` on a -page predating this rule. Given `--page`, both are written straight onto the -target page (once - fill-once, like `touch`, see `touch._capture_field_or_fail`); -without `--page` there is no page yet to write them onto (`wiki-ingest` -creates it afterwards), so this command instead prints the exact -`wikitool new source --set fidelity=... --set authority=...` follow-up line, -and `new source` itself refuses to scaffold a source page without both. -`--replaces` is the one path where both become *optional*: passing them there -is the sanctioned way to correct an already-set capture value on a later -edition, the same way `touch`'s own refusal points back to `--replaces`. - -Multi-owner raw files (`provenance.duplicate_raw_file_owners`) are refused -rather than silently moved: relocating a file another page also claims would -break that page's `raw_files:` without it ever being consulted. - -**Stem uniqueness across the whole of `raw/`** (Gitea #64, widened by #67): -without it, a second, unrelated source whose primary file happens not to -collide on the exact filename slips silently into an existing bundle, because -the per-file `dst.exists()` check above never looks at the bundle directory -itself. The set of names occupied anywhere under `raw/` - file stems and -bundle directory names alike, at whichever level directly holds files - must -stay unique; `_occupied_stems()` and the check built on it enforce that, -while still allowing a call to grow a bundle it already owns (via `--page`, -or by continuing an existing bundle). It used to be scoped to one type -directory; #67 removes type directories from the addressing scheme entirely, -so uniqueness now has to span old-style flat directories and the new date -shard together, or `--replaces` on a bygone stem would be ambiguous across -shards. - -**`--replaces `** is the only sanctioned way past that rule: whether -a new file is a later edition of an existing source or a second, separate one -is a human's decision, never the tool's or an agent's, so the command refuses -and names both routes rather than choosing one (Gitea #64 decision 2). -""" -from __future__ import annotations - -import datetime -import json -import os -import re -import shutil -import tempfile -from dataclasses import dataclass -from pathlib import Path -from typing import Annotated, Optional - -import typer -from rich.markup import escape - -from chemenu import cli_contract, config, filters, guideline_export, repo_capture, web_capture -from chemenu.commands._util import fail, path_budget_problem_for, rel_path, success -from chemenu.errors import BackendError, ChemenuError, ValidationError -from chemenu.frontmatter_io import write_page -from chemenu.kb_scan import load_kb_pages -from chemenu.provenance import citing_pages, source_pages_by_raw_file, source_raw_files -from chemenu.type_resolver import resolver -from chemenu.version import VersionError, read_version - -app = typer.Typer(help="Bring raw material into incoming/, and promote it from there into raw/.") - -# The type this command always promotes into eventually - hardcoded rather -# than derived from --page, because there is no page yet in the common case -# (see module docstring): the follow-up hint and the capture-field/enum -# validation below both need *a* type to read from, and `source` is the only -# one `raw accept` has ever scaffolded towards. -_SOURCE_TYPE_PATH = "types/source.md" - - -def _incoming_dir() -> Path: - return config.ROOT / "incoming" - - -def _resolve(raw: Path) -> Path: - return raw if raw.is_absolute() else config.ROOT / raw - - -def _shard_dir(today: Optional[datetime.date] = None) -> Path: - """`raw///`, computed from the accept date - never from a count, - so it can never rebalance and break a `[^cite-id]` anchor (Gitea #67). - """ - d = today or datetime.date.today() - return config.RAW_DIR / f"{d.year:04d}" / f"{d.month:02d}" - - -class _Refused(Exception): - """A check that runs before anything moves has failed; the message is what - `fail()` prints. Raised rather than failing on the spot so `raw pending` - can ask the same checks of every candidate without exiting - one set of - checks, two callers (Gitea #112).""" - - -def _refusals_fail(fn, *args): - try: - return fn(*args) - except _Refused as exc: - fail(escape(str(exc))) - - -def _check_directly_in_incoming(path: Path, incoming: Path) -> None: - """`path` must sit directly in `incoming/`. - - Up to Gitea #112 one subdirectory level was tolerated and ignored, so an - old `incoming//` habit kept working after #67 stopped reading the - type from it. A subdirectory is a source of its own now - accepted as a - whole - so a file inside one is refused with both ways out named. - """ - try: - rel = path.relative_to(incoming) - except ValueError: - raise _Refused( - f"{rel_path(path)} is not under incoming/ - `raw accept` and `raw fetch --html` " - "only take files from there. See raw/CONTRACT.md." - ) from None - if len(rel.parts) < 1: - raise _Refused(f"incoming/{rel.as_posix()} names no file.") - if len(rel.parts) > 1: - top = rel.parts[0] - raise _Refused( - f"incoming/{rel.as_posix()} lies in a subdirectory of incoming/ - a file is taken " - "only from directly inside incoming/, and a subdirectory is a source of its own. " - f"Either accept the whole folder as one source (raw accept incoming/{top}), or move " - "the file up into incoming/ and accept it there. See raw/CONTRACT.md." - ) - - -def _validate_under_incoming(path: Path, incoming: Path) -> None: - _refusals_fail(_check_directly_in_incoming, path, incoming) - - -_YEAR_DIR = re.compile(r"\d{4}") - - -def _occupied_stems(raw_dir: Path) -> dict[str, Path]: - """Every name occupied anywhere under `raw/`: file stems and bundle - directory names alike, at whichever level directly holds files. - - Distinguishes the pre-#67 flat layout (`raw//`) - from the #67 date shard (`raw///`) structurally, - by whether a top-level directory name is a 4-digit year - not from a - hardcoded list of legacy type names, so this keeps working unchanged if a - legacy directory is ever renamed by hand. - """ - occupied: dict[str, Path] = {} - if not raw_dir.is_dir(): - return occupied - for top in raw_dir.iterdir(): - if not top.is_dir(): - continue # raw/CONTRACT.md - if _YEAR_DIR.fullmatch(top.name): - for month in top.iterdir(): - if month.is_dir(): - occupied.update(_leaf_group_entries(month)) - else: - occupied.update(_leaf_group_entries(top)) - return occupied - - -def _leaf_group_entries(group_dir: Path) -> dict[str, Path]: - """The addressable entries directly inside one leaf-group directory - (a legacy type dir, or one `raw///`): a lone file occupies its - stem, a bundle directory occupies its own name.""" - return {entry.stem if entry.is_file() else entry.name: entry for entry in group_dir.iterdir()} - - -def _stem_collision_message(claimed_name: str, holder: Path) -> str: - example_target = holder - if holder.is_dir(): - children = sorted(holder.iterdir()) - example_target = children[0] if children else holder - return ( - f'{rel_path(holder)} already claims the stem "{claimed_name}" under raw/.\n' - " These are two different intents and only you can tell them apart:\n" - f" Same source, new edition -> tools/wikitool raw accept " - f"--replaces {rel_path(example_target)} \n" - " A second, separate source -> rename it in incoming/ (add a distinguishing " - "suffix) and accept it normally\n" - " raw accept does not guess which one this is." - ) - - -def _check_files(resolved: list[Path]) -> None: - incoming = _incoming_dir() - for path in resolved: - if not path.is_file(): - raise _Refused(f"{rel_path(path)} does not exist or is not a file.") - _check_directly_in_incoming(path, incoming) - if path.name == repo_capture.MANIFEST_NAME: - raise _Refused(_reserved_manifest_message(path)) - _check_no_export(resolved) - names = [path.name for path in resolved] - if len(names) != len(set(names)): - raise _Refused("Two files share a filename; rename one before promoting.") - - -def _check_no_export(paths: list[Path]) -> None: - """Refuse a file a wiki exported (`export guidelines`): it is generated - from `kb/`, so taking it into `raw/` would feed the wiki its own output - back as a source. `raw capture` already leaves such a file out; this - closes the way in through `incoming/`.""" - exports = [p for p in paths if guideline_export.file_is_export(p)] - if exports: - listed = "\n".join(f" - {rel_path(p)}" for p in exports) - raise _Refused( - "These files are a wiki's guideline export - their first line starts with " - "` markers and a WARN when it yields almost no text (a scan; there is " - "no OCR); an encrypted PDF is refused. A `.msg` is a container like an `.eml`; an " - "embedded message inside it is named, not read. The text of a mail (`.eml` and `.msg` " - "alike) is its plain text, unless that is under 10% of the length of the HTML text - a " - "\"plain text version not available\" placeholder - in which case it is the HTML text " - "and a WARN names both lengths. The other version of the text in a " - "`multipart/alternative` is not an attachment.", - "A workbook (`.xlsx`, `.xlsm`) gets its cells as files: beside its `.md` a folder " - "`.sheets/` (`derived.sheets/` for a single file) holds one `NN-.tsv` " - "per sheet - tab-separated per RFC 4180, UTF-8, row 1 of the file is row 1 of the sheet " - "- and, for a sheet with formulas, `NN-.formulas.tsv` listing cell, formula and " - "last stored value. The `.md` is the overview: one section per sheet with its range, " - "the file names and a preview. Formulas are not recalculated; a shared formula is " - "written out per cell, and one that cannot be is named `shared:` with a " - "WARN. The folder counts against the path budget and is accepted with the rest; an " - "existing one is never overwritten.", - "A ZIP and a mail are containers. A ZIP at the top of `incoming/` becomes the folder " - "`incoming//` holding its files under their own names, each with a `.md` " - "where a filter reads it; the ZIP is removed and the output names it with its sha256. A " - "mail becomes a folder with `original.eml`, `derived.md` and its attachments under their " - "own names; a ZIP attached to it is unpacked once into a folder of its own, and an " - "attached mail is quoted in `derived.md`, not unpacked. A ZIP inside a ZIP, an " - "absolute or `..` path, a symbolic link, an encrypted entry, more than 1000 entries, a " - "file over 25 MiB or more than 250 MiB in total refuse the whole call. Hidden entries " - "(`__MACOSX/`, `.DS_Store`, anything starting with a dot) are named, not written. A name " - "taken twice, or reserved (`original.*` and `derived.md` at the top of a mail's folder, " - "`_capture.json` anywhere), gets `-2`, `-3` before its extension, and the output lists " - "each rename.", - "A name already taken under `raw/` is a WARN, not a refusal: whether the folder is a " - "later edition of that source or a separate one is the human's call. A later edition of " - "a derived source is replaced file by file with `raw accept --replaces`; there is no " - "`--replaces-bundle` for it.", - "After a success the output prints the warnings the filters raised, at most three " - "`CHECK` lines per format that was derived in this run - what to look at before the " - "text is trusted - and the `raw accept` line for the result.", - "Never edit `derived.md` by hand: a better derivation is a new `raw derive` of the original.", - ), - failures=( - cli_contract.Failure( - cause="The path is not directly in `incoming/`, does not exist, or no filter reads " - "the file's extension", - reaction="Nothing was written. Fix the argument and retry once - a file in a " - "subfolder is moved up, or its folder is derived whole", - ), - cli_contract.Failure( - cause="The target folder, a `.md` or a workbook's `.sheets` folder already exists, or the path would be over the " - "path budget", - reaction="Nothing was written. Remove what is there or pass `--name `, " - "then retry once. An existing `.md` is never overwritten", - ), - cli_contract.Failure( - cause="The document cannot be read (damaged, encrypted), a container breaks one of its " - "rules (a ZIP inside a ZIP, an unsafe path, a limit) or the filter's package is not " - "installed", - reaction="Nothing was written. A missing package is reported with the preflight " - "hint - show it to the user and stop, never install it. Any other filter keeps " - "working", - ), - ), - examples=( - "tools/wikitool raw derive incoming/report.docx", - "tools/wikitool raw derive incoming/Angebot-XYZ.xlsx --name angebot-xyz", - "tools/wikitool raw derive incoming/meeting-files", - "tools/wikitool raw derive incoming/scans.zip", - ), - never=( - "Never edit `derived.md` or its header by hand.", - "Never derive a file the user did not put into `incoming/` for this purpose.", - ), - see_also=( - "`raw/CONTRACT.md` \"Getting a document in: `raw derive`\" - the rules and why", - "`wikitool raw accept` - promotes the folder into `raw/`", - "`instructions/wiki-ingest/SKILL.md` - where a PDF, mail or Office file to ingest starts", - ), -)) -@app.command("derive") -def raw_derive_command( - path: Path = typer.Argument(..., help="A file or a folder directly in incoming/"), - name: Annotated[Optional[str], typer.Option( - "--name", help="Folder stem for a single file instead of the file's own stem" - )] = None, -): - """Derive reading text from a document in incoming/ and keep the original. - See raw/CONTRACT.md "Getting a document in: `raw derive`".""" - resolved = _resolve(path) - if resolved.is_dir(): - if name is not None: - fail("--name applies to a single file; a folder keeps its own name.") - derived, untouched = _refusals_fail(_plan_derive_folder, resolved) - _write_exclusive([write for d in derived for write in [(d.target, d.markdown), *d.extras]]) - lines = [f"Derived {len(derived)} file(s) in {rel_path(resolved)}/:"] - for d in derived: - more = f" + {len(d.extras)} file(s) in {rel_path(d.extras[0][0].parent)}/" if d.extras else "" - lines.append(f" {rel_path(d.source)} -> {rel_path(d.target)} ({d.filter.name}){more}") - if untouched: - lines.append( - " No filter for, left as they are: " + ", ".join(rel_path(p) for p in untouched) - ) - success(escape("\n".join(lines))) - warning = _collision_warning(resolved.name) - if warning: - typer.echo("WARN " + warning) - _echo_notes([d.derivation for d in derived]) - typer.echo(_derive_next(resolved)) - return - - plan = _refusals_fail(_plan_derive_file, resolved, name) - _write_folder(resolved, plan) - container = plan.container - if plan.original is None: - success(escape( - f"Unpacked {resolved.name} into {rel_path(plan.folder)}/ ({plan.candidate.name}); " - f"{resolved.name} is removed." - )) - elif container is not None and container.files: - success(escape( - f"Derived {rel_path(plan.derived)} from {resolved.name} ({plan.candidate.name}) with " - f"{len(container.files)} file(s) from its attachments; the original is now " - f"{rel_path(plan.original)}." - )) - else: - success(escape( - f"Derived {rel_path(plan.derived)} from {resolved.name} ({plan.candidate.name}); the " - f"original is now {rel_path(plan.original)}." - )) - if plan.extras: - lines = f" + {len(plan.extras)} file(s) beside it in {rel_path(plan.folder)}/{plan.extras[0][0].rsplit('/', 1)[0]}/" - typer.echo(escape(lines)) - if container is not None: - report = _container_report(container) - if report: - typer.echo("\n".join(report)) - warning = _collision_warning(plan.stem) - if warning: - typer.echo("WARN " + warning) - _echo_notes(plan.derivations, container) - typer.echo(_derive_next(plan.folder)) - - -# --- raw capture / raw status ------------------------------------------------- -# -# Documentation from a git repository, as one bundle per repository and ref -# rule. The git side - resolving, fetching, selecting, the manifest - lives in -# `chemenu.repo_capture`; these adapters decide where the files go and what the -# caller is told. Like `raw fetch`, `raw capture` ends in `incoming/`, never in -# `raw/`: promoting stays `raw accept`'s job. - - -def _unusable_name(name: str) -> bool: - """Not one plain directory name: empty, padded, hidden, a path, a control - character, or the manifest's reserved name.""" - return ( - not name.strip() - or name != name.strip() - or name.startswith(".") - or name == repo_capture.MANIFEST_NAME - or any(c in name for c in "/\\:") - or any(ord(c) < 0x20 for c in name) - ) - - -def _check_bundle_name(name: str) -> None: - if _unusable_name(name): - fail(escape( - f"--name {name!r} is not a usable bundle name: one directory name - no path separator, " - f"not hidden, not {repo_capture.MANIFEST_NAME}." - )) - - -def _captured_raw_bundle(path: Path) -> tuple[Path, repo_capture.Manifest]: - bundle = _resolve(path) - try: - bundle.relative_to(config.RAW_DIR) - except ValueError: - fail(escape(f"--update {rel_path(bundle)} does not lie under raw/.")) - manifest_path = bundle / repo_capture.MANIFEST_NAME - if not manifest_path.is_file(): - fail(escape( - f"--update {rel_path(bundle)} is not a captured bundle - it has no " - f"{repo_capture.MANIFEST_NAME}. Only a bundle raw capture wrote can be updated." - )) - try: - return bundle, repo_capture.read_manifest(manifest_path) - except ValidationError as exc: - fail(escape(f"{rel_path(manifest_path)}: {exc}")) - - -def _write_bundle(target: Path, files: dict[str, bytes], manifest: repo_capture.Manifest) -> None: - """Create `target` with every file and the manifest, or nothing at all: - written into a fresh hidden directory beside it (which `raw pending` - skips), then renamed into place in one step. A failure removes only that - directory, which this call created and nothing else wrote into.""" - target.parent.mkdir(parents=True, exist_ok=True) - staging = Path(tempfile.mkdtemp(prefix=f".{target.name}-", dir=target.parent)) - try: - for rel, data in files.items(): - path = staging / rel - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("xb") as handle: - handle.write(data) - (staging / repo_capture.MANIFEST_NAME).write_text( - manifest.to_json(), encoding="utf-8", newline="\n" - ) - os.rename(staging, target) - except OSError as exc: - shutil.rmtree(staging, ignore_errors=True) - fail(escape(f"Could not write {rel_path(target)}/: {exc}. Nothing was left in incoming/.")) - - -@cli_contract.record(cli_contract.CommandRecord( - path="raw capture", - summary="Capture documentation from a git repository into `incoming//`, with a " - "manifest naming the repository, ref rule and commit, for `raw accept` to promote.", - synopsis=( - cli_contract.Variant( - usage="raw capture --ref --path [--path " - "...] --name --fidelity --authority ", - notes="First capture of a repository", - ), - cli_contract.Variant( - usage="raw capture --update [--fidelity ] [--authority ]", - notes="Capture the current state of an already-accepted bundle, for `raw accept " - "--replaces-bundle`", - ), - ), - properties=cli_contract.Properties( - effect=cli_contract.Effect.WRITE, - idempotent=cli_contract.Idempotent.NO, - atomic="Yes for what it leaves in `incoming/` - the bundle is written into a hidden " - "staging directory and renamed into place in one step; a failure removes the staging " - "directory. The git cache under `tools/.wikitool_capture/` is updated either way", - budget=cli_contract.Budget.COUNTED, - network=cli_contract.Network.YES, - ), - notes=( - "Writes into `incoming//` only, never into `raw/`: `raw accept incoming/` " - "promotes it, as one folder, to `raw////`.", - "`--ref` is a branch name (`main`) or a tag pattern (`v*`, any of `*?[`), which names the " - "newest matching tag by version order. It resolves to one commit - for an annotated tag, " - "the commit it points at.", - "The commit is fetched by ref name, shallowly, into a bare cache repository per URL under " - "`tools/.wikitool_capture/` (gitignored). Files are read from it as blobs, never through a " - "checkout, so they are byte-identical to the repository's - no line-ending conversion, no " - "filter.", - "`--path` globs (repeatable) select files by their path in the repository, as git's " - "`:(glob)` pathspec does: `*`, `?` and `[...]` do not cross `/`; `**` as a whole segment " - "spans any number of directories; a glob without wildcards also takes everything below it " - "as a directory. Files keep their repository paths inside the bundle.", - "Never captured, each named in the output with its reason: a file whose first line starts " - "with `