diff --git a/.gitignore b/.gitignore index 7555c3a..e790f5e 100644 --- a/.gitignore +++ b/.gitignore @@ -180,18 +180,17 @@ npm-debug.log* /incoming/* !/incoming/.gitkeep -# MCP `submit` tool quarantine (see raw/CONTRACT.md "Getting a file in from -# outside" and tools/chemenu/upload.py). Material pushed by a caller that is -# not this terminal, before a human has reviewed any of it - stronger than -# `incoming/` above: not merely uncommitted, but read by no command in the +# The quarantine of the intake channels (see raw/CONTRACT.md "Getting a file in +# from outside" and tools/chemenu/upload.py). Material that came from a caller +# that is not this terminal - the MCP `submit` tool, a synced folder, a program +# of this checkout - before a human has reviewed any of it: stronger than +# `incoming/` above, not merely uncommitted but read by no command in the # ordinary pipeline. Unlike `incoming/` above (Gitea #88), this one keeps the -# directory form and gets no `.gitkeep`: the directory is created on demand -# by the one function that is allowed to write into it, and a checkout that -# never arms the `submit` tool never gets one - there is no fresh-clone case -# to cover here, since nothing reads this path before that function creates -# it. Never anchored back open by the content backstop below, same as -# `incoming/`. -/mcp-upload/ +# directory form and gets no `.gitkeep`: the directory is created on demand by +# the one function that is allowed to write into it, together with its own +# `.gitignore`, so a checkout that never arms a channel never gets one. Never +# anchored back open by the content backstop below, same as `incoming/`. +/quarantine/ # Content backstop - keep this block last. Nothing under raw/, kb/, work/ or sites/ may # be excluded by a pattern above; see the header note for why directory patterns diff --git a/CHANGES.md b/CHANGES.md index adc7a0d..5fa3b35 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,17 +14,20 @@ place only: `instructions/dev/version-parts.md` § The candidate model. --- -## 10.0.0-beta.10 - 2026-10-12 - Backup in einen Sync-Ordner (OneDrive): backup init legt ein Bare-Repo unter /git als origin an, backup run und publish spiegeln die Maschinendaten nach //, doctor warnt vor .git im Sync-Ordner +## 10.0.0-beta.11 - 2026-10-12 - Eingangskanäle: der intake-Abschnitt kennt mehrere Kanäle (mcp, folder, command), alle enden in quarantine/; neues Kommando upload collect zieht Sync-Ordner und Programme in die Quarantäne **Author:** Torben Nehmer -**Breaking Change:** Die vier Dateien .wikitool-remotes.json, .wikitool-telemetry.json, .wikitool-upload.json und .wikitool-tasks.json werden nicht mehr gelesen; ein Kommando, das die Einstellung braucht, endet mit Exit 42, bis 'tools/wikitool config migrate' sie in wikitool.json, .wikitool-local.json und die Secrets-Referenzen übernommen hat +**Breaking Change:** +- Die vier Dateien .wikitool-remotes.json, .wikitool-telemetry.json, .wikitool-upload.json und .wikitool-tasks.json werden nicht mehr gelesen; ein Kommando, das die Einstellung braucht, endet mit Exit 42, bis 'tools/wikitool config migrate' sie in wikitool.json, .wikitool-local.json und die Secrets-Referenzen übernommen hat +- Die Quarantäne heißt quarantine/ statt mcp-upload/. Eine Instanz mit wartenden Einreichungen verschiebt das Verzeichnis einmal von Hand (Migrationsschritt); bis dahin enden die upload-Kommandos mit Exit 1. **Migration:** required - instructions/migrations/10.0.0-settings-tiers.md **High impact** - Die Einzeldateien .wikitool-remotes/-telemetry/-upload/-tasks.json sind in den drei Einstellungsstufen aufgegangen; neues Kommando config migrate +- Eingangskanäle: der intake-Abschnitt kennt mehrere Kanäle (mcp, folder, command), alle enden in quarantine/; neues Kommando upload collect zieht Sync-Ordner und Programme in die Quarantäne **Medium impact** - Einstellungen in drei Stufen: wikitool.json (Instanz), .wikitool-local.json (Checkout), Secrets nur per Verweis - auch über die 1Password-CLI; neues Kommando config set @@ -63,6 +66,10 @@ Der Abschnitt `fetch` der Einstellungen ordnet einem Host einen Abrufweg zu. Bei Die vier Einzeldateien `.wikitool-remotes.json`, `.wikitool-telemetry.json`, `.wikitool-upload.json` und `.wikitool-tasks.json` gibt es nicht mehr; ihre Felder sind die Abschnitte `remotes`, `telemetry`, `upload`, `tasks`, `superproductivity` und `caldav` der drei Stufen. Zugangsdaten sind dort nur noch Verweise. Der Code liest die alten Dateien nicht mehr: ein Kommando, das die Einstellung braucht, endet mit Exit 42 und nennt `tools/wikitool config migrate`, `doctor` meldet FAIL, der Telemetrie-Hook bleibt still. `config migrate` läuft einmal je Checkout, ist wiederholbar und lässt bei jedem Abbruch die alten Dateien unverändert. `WIKITOOL_TASKS_CONFIG` zeigt auf ein Profil im Format von `.wikitool-local.json`. Nur `.wikitool-tools.json` bleibt eine eigene Datei. +### Eingangskanäle: mehrere Wege in dieselbe Quarantäne (#213) + +Der Abschnitt `intake` in `.wikitool-local.json` ersetzt `upload` und führt benannte Kanäle mit einem Handler: `mcp` (das `submit`-Werkzeug, höchstens einer), `folder` (ein Ordner außerhalb der Instanz, den ein Sync-Client füllt) und `command` (ein Programm aus `commands`). Alle enden in `quarantine/` (vorher `mcp-upload/`), jede Einreichung trägt `channel`, `handler` und `origin`, und `upload accept` bleibt das Gate. Neu ist `upload collect [...]`: es zieht Ordner und Programme in die Quarantäne, entfernt eine Quelle erst, wenn die Kopie fertig ist und die Quelle noch Größe und sha256 hat, und lässt alles andere liegen. `config migrate` macht aus `.wikitool-upload.json` den Kanal `mcp`; wartende Einreichungen in `mcp-upload/` verschiebt der Migrationsschritt von Hand, bis dahin enden die upload-Kommandos mit Exit 1. + ### Backup in einen Sync-Ordner (#186) Ohne Git-Server bleibt der Arbeitsbaum außerhalb des Sync-Ordners; gesichert wird in das Ziel, das die Sektion `backup` von `.wikitool-local.json` nennt. `backup init` legt `/git` als Bare-Repo an und macht es zu `origin`, wenn keins existiert. Mit `machine_data: true` spiegelt jedes `publish` nach dem Push `.wikitool-local.json`, `ENVIRONMENT.md`, `incoming/` und die `extra`-Pfade nach `//` samt Manifest; `backup run` tut dasselbe von Hand. Geschrieben wird nur dort und atomar, gelesen nur innerhalb der Instanz, Secrets nie. Eine gescheiterte Sicherung nach dem Push ist eine `WARN`-Zeile mit Exit 0, eine fehlerhafte Sektion stoppt `publish` vor dem Commit. `doctor` meldet `sync-folder` und `backup`. Ablauf und Wiederherstellung: `docs/how-to/back-up-to-a-synced-folder.md`. diff --git a/VERSION b/VERSION index d7b4f75..73429ef 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -10.0.0-beta.10 +10.0.0-beta.11 diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md index 2ee6090..fb3bb74 100644 --- a/docs/explanation/architecture.md +++ b/docs/explanation/architecture.md @@ -81,9 +81,10 @@ chemenu/ │ ├── ingest-large-tree.md │ ├── ingest-queue.md # Reviewing a submission before wikitool upload accept promotes it │ └── wiki-*/SKILL.md # Skills - copied into .agents/skills/ and .claude/skills/ -├── mcp-upload/ # QUARANTINE (optional): the MCP `submit` tool's write path, gitignored - -│ # read by no command in the ordinary pipeline; a human reviews it with -│ # `wikitool upload list/show/accept/reject` +├── quarantine/ # QUARANTINE (optional): where the intake channels (MCP `submit`, a synced +│ # folder, a command) put what nobody has reviewed, gitignored - read by +│ # no command in the ordinary pipeline; a human reviews it with +│ # `wikitool upload list/show/accept/reject`, `upload collect` fills it ├── incoming/ # INBOX: content gitignored - drop a file or one folder per source here, `raw accept` promotes it ├── raw/ # INPUT: immutable, untrusted source material │ ├── CONTRACT.md # Date shard, capture fields, immutability, untrusted content diff --git a/docs/explanation/pipeline-rationale.md b/docs/explanation/pipeline-rationale.md index 5a7a968..b554ff2 100644 --- a/docs/explanation/pipeline-rationale.md +++ b/docs/explanation/pipeline-rationale.md @@ -19,9 +19,11 @@ not yet reviewed and not yet a citation target, so its being gitignored and read ingest session does not weaken anything - the boundary is the promotion into `raw/` itself, not the moment a human happened to drop a file somewhere. -Once a document can arrive from *outside* - the MCP server's optional `submit` tool - "a human -happened to drop a file somewhere" stops describing how everything reaches `incoming/`, so the -near side of the boundary gets a stage of its own rather than a second meaning. `mcp-upload/` +Once a document can arrive from *outside* - the MCP server's optional `submit` tool, a folder a +sync client fills, a program of the checkout - "a human happened to drop a file somewhere" stops +describing how everything reaches `incoming/`, so the near side of the boundary gets a stage of its +own rather than a second meaning. Every such channel ends in the same `quarantine/`, because what +the boundary protects against is the lack of a reviewer, not the transport. `quarantine/` holds what nobody has looked at yet; `incoming/` holds what someone has. Two arrows, two different things being granted: `upload accept` grants *trust* (a human read the material and took responsibility for it), `raw accept` grants *immutability* (it becomes a citation target diff --git a/docs/how-to/README.md b/docs/how-to/README.md index 3281b15..77712b4 100644 --- a/docs/how-to/README.md +++ b/docs/how-to/README.md @@ -30,6 +30,8 @@ Running the instance: - [Trace and score agent sessions](trace-and-score-sessions.md) - [Use 1Password for settings](use-1password-for-settings.md) - secrets and private values out of the tree, on the desktop and in CI +- [Collect documents from a synced folder](collect-from-a-synced-folder.md) - a folder OneDrive and + the like fill, or a program of your own, as an intake channel into the quarantine - [Back up to a synced folder](back-up-to-a-synced-folder.md) - OneDrive and the like as the backup of an instance without a git server, and restoring from it diff --git a/docs/how-to/back-up-to-a-synced-folder.md b/docs/how-to/back-up-to-a-synced-folder.md index f11211a..8016171 100644 --- a/docs/how-to/back-up-to-a-synced-folder.md +++ b/docs/how-to/back-up-to-a-synced-folder.md @@ -100,7 +100,7 @@ push. | `ENVIRONMENT.md` | yes, if present | written by hand | | Everything in `incoming/` but `.gitkeep` | yes | sources waiting there are backed up nowhere else | | The `extra` paths | yes | what you named | -| `mcp-upload/` | no | material submitted from outside that nobody has reviewed yet | +| `quarantine/` | no | material that arrived through an intake channel and that nobody has reviewed yet | | `reports/` | no | regenerated by `lint`; telemetry holds prompts | | `.wikitool-tools.json`, `tools/.venv/`, the published skills | no | the preflight and `instructions sync` rebuild them | | The secrets file | **never** | it lies outside the instance, and a backup reads nothing outside it | diff --git a/docs/how-to/collect-from-a-synced-folder.md b/docs/how-to/collect-from-a-synced-folder.md new file mode 100644 index 0000000..bd08a3d --- /dev/null +++ b/docs/how-to/collect-from-a-synced-folder.md @@ -0,0 +1,133 @@ +# Collect documents from a synced folder + +Documents that are not typed or fetched at this terminal can reach the wiki through an intake +channel. Besides the MCP `submit` tool there are two channels that this checkout *pulls* from: a +folder that a sync client such as OneDrive fills (handler `folder`), and a program of your own +(handler `command`). Both end in the quarantine `quarantine/`, exactly like a submission through the +server, and both wait there for a human to run `wikitool upload accept` - the Upload Review Gate +stands between every channel and `incoming/`. + +This guide creates a `folder` channel, explains what `upload collect` leaves alone and why, and shows +a `command` channel with an example script. The field reference is in the +[configuration reference](../reference/configuration.md#intake-channels); the review is in +[ingesting a source](ingest-a-source.md#a-document-submitted-from-outside). + + +## Contents + +- [Create the folder channel](#create-the-folder-channel) +- [What stays in the folder, and why](#what-stays-in-the-folder-and-why) +- [`settle_seconds` and what it cannot promise](#settle_seconds-and-what-it-cannot-promise) +- [Deleting in a synced folder deletes everywhere](#deleting-in-a-synced-folder-deletes-everywhere) +- [A command channel](#a-command-channel) +- [Afterwards](#afterwards) + + +## Create the folder channel + +Channels are not written by `config set`: every field of the `intake` section opens a path into the +checkout, so it is yours to edit by hand in `.wikitool-local.json`. + +```json +{ + "intake": { + "channels": { + "onedrive": { + "handler": "folder", + "path": "/home/you/OneDrive/wiki-inbox", + "settle_seconds": 60, + "max_bytes": 10485760, + "allowed_extensions": [".pdf", ".docx", ".md", ".txt"] + } + } + } +} +``` + +`path` has to be absolute and lie outside the instance. Check it with `tools/wikitool doctor`, then +pull: + +```bash +tools/wikitool upload collect # every pull channel +tools/wikitool upload collect onedrive # only this one +tools/wikitool upload list +``` + +A collected file is stored unchanged in `quarantine//` with a manifest that names the channel, +the handler and the original path. A folder knows nobody who dropped a file, so the submitter is +recorded as `unknown`, with the handler as its source: do not trust the name or content of such a +file any more than that of a stranger's. + +## What stays in the folder, and why + +`upload collect` reads the top level of the folder only, and it removes one kind of thing: a file it +has just stored. Everything else stays where it is and is reported with its reason: + +| Entry | Why it stays | +|---|---| +| A subdirectory | Nothing descends into it, and nothing deletes a directory. | +| A name starting with `.` or `~$`, a hidden or system file | Editors and sync clients keep temporary files like these next to the real one. | +| A symbolic link | Following it could leave the folder. | +| An extension outside `allowed_extensions`, or a file above `max_bytes` | The same limits as for `submit`. | +| A file younger than `settle_seconds` | It may still be arriving. | +| A file that changed while it was being read or copied | The source is removed only if it still has the size and sha256 that were stored. | + +A file whose sha256 already waits in the quarantine is reported as a duplicate, with the id of the +existing submission, and is removed from the folder. + +## `settle_seconds` and what it cannot promise + +A sync client writes a file in pieces, so a file that is half there looks like a complete, smaller +one. `settle_seconds` (default 60) skips files whose modification or creation time is younger than +that, and the check that the source did not change during the copy catches the rest. Neither can see +a client that pauses for longer than the interval in the middle of a file and then resumes: such a +file is stored incomplete, and the review is where you notice it. Raise the value if your client +works over a slow link. + +## Deleting in a synced folder deletes everywhere + +A file removed from a folder that OneDrive syncs is removed on every device that syncs it. That is +what a collected file should do - it is now in the quarantine - but it also means a collect on one +machine empties the folder for all of them. The client's recycle bin keeps the file for a while; the +quarantine copy, which is never synced, is the one to rely on. Collect from one machine only. + +## A command channel + +A `command` channel runs a program of your own, which fetches documents from somewhere that has no +synced folder - a mailbox, a share, a scanner. The program is named in `commands` and referenced from +the channel; the section stays valid only if every reference resolves. + +```json +{ + "intake": { + "commands": { + "scan-mail": ["/home/you/bin/fetch-attachments.sh"] + }, + "channels": { + "mail": { "handler": "command", "command": "scan-mail" } + } + } +} +``` + +The program is started without a shell and with stdin closed, with an empty staging directory under +`quarantine/.staging/` appended as its last argument. It has 300 seconds. It leaves the files it +wants to hand over directly in that directory: + +```bash +#!/bin/sh +# fetch-attachments.sh +set -eu +out="$1" +rclone copy remote:wiki-inbox "$out" --max-age 1d +``` + +A program that exits non-zero, does not start or times out fails its channel: nothing is taken, the +staging directory is removed, and the last 2000 characters of its stderr are shown. Other channels +still run, and `upload collect` exits 1 if any channel failed. With no pull channel configured it +exits 0 with "nothing to collect". + +## Afterwards + +The agent does the same at the start of an ingest when you name no file: `upload collect`, then +`upload list`, then a report of what waits. It never runs `upload accept` for you. diff --git a/docs/how-to/ingest-a-source.md b/docs/how-to/ingest-a-source.md index ddd70cd..33bc70f 100644 --- a/docs/how-to/ingest-a-source.md +++ b/docs/how-to/ingest-a-source.md @@ -93,11 +93,14 @@ Every ten sources, the agent suggests a lint run - see [Maintain the wiki](maint ## A document submitted from outside When the [MCP read server](set-up-the-mcp-server.md) has its optional `submit` tool turned on, -documents can arrive from another machine. They land in the quarantine `mcp-upload/`, not in +documents can arrive from another machine. A folder a sync client fills and a program of the +checkout are two more ways in ([Collect documents from a synced +folder](collect-from-a-synced-folder.md)). All of them land in the quarantine `quarantine/`, not in `incoming/`, and nothing else reads them there. Before an ingest can start, a human has to promote the submission: ```bash +tools/wikitool upload collect # pull the folder and command channels into the quarantine tools/wikitool upload list # what is waiting tools/wikitool upload show # one submission's manifest: who sent it, size, checksum ``` diff --git a/docs/how-to/set-up-the-mcp-server.md b/docs/how-to/set-up-the-mcp-server.md index 90c231f..5052dea 100644 --- a/docs/how-to/set-up-the-mcp-server.md +++ b/docs/how-to/set-up-the-mcp-server.md @@ -160,18 +160,19 @@ reviewed yet. ## Optional: the submit path -Without this, `submit` does not exist as a tool - not unused, but not registered. An -`upload` section in the served checkout's `.wikitool-local.json` turns it on. Its fields, defaults and what a malformed -section does are in the [configuration reference](../reference/configuration.md#mcp-submissions); a -broken section never means "no limits", it stops the server from starting. +Without this, `submit` does not exist as a tool - not unused, but not registered. A +channel with the handler `mcp` in the `intake` section of the served checkout's +`.wikitool-local.json` turns it on. Its fields, defaults and what a malformed section does are in +the [configuration reference](../reference/configuration.md#intake-channels); a broken section never +means "no limits", it stops the server from starting. **With `submit` on, the middleware has a second duty.** It has to set the identity header itself, on every authenticated request, and discard any copy the client sent. The server trusts that header as the value for `submitter`. A header the client could set itself would be a claim, not an identity. -If `identity_header` names a header the middleware does not set, every submission is refused for +If the channel's `identity_header` names a header the middleware does not set, every submission is refused for lack of an identity. -Submitted files land in `mcp-upload//`, which is gitignored and read by no other command. From +Submitted files land in `quarantine//`, which is gitignored and read by no other command. From there a human promotes or rejects them, in the same checkout as the server rather than in its process. That review is part of [ingesting a source](ingest-a-source.md#a-document-submitted-from-outside). @@ -228,7 +229,7 @@ the client starts a different Python than the instance's. Give the client the ab **`"commit": null` in every answer**: the served tree has uncommitted changes. Either the sync is not running, or something writes into the corpus that has no business there. The five read tools never -write. `submit`, when it is on, writes only to `mcp-upload/`, which is gitignored and so is never a +write. `submit`, when it is on, writes only to `quarantine/`, which is gitignored and so is never a cause of `"commit": null` either. **`commit` names an old revision**: the sync timer is not running. @@ -246,6 +247,6 @@ Deployment specifics - the cluster, ingress hosts, the secret store, the GitOps private infrastructure, and this repository is public. There is no container image for the server yet. Until there is, the path above - the venv, -`python -m chemenu.mcp`, a proxy in front - is the complete one. With `submit` on, `mcp-upload/` +`python -m chemenu.mcp`, a proxy in front - is the complete one. With `submit` on, `quarantine/` has to survive restarts the same way the rest of the checkout does. Otherwise a submitted file that nobody has reviewed yet loses its quarantine. diff --git a/docs/reference/commands/README.md b/docs/reference/commands/README.md index 3ed7ab4..dbf8f99 100644 --- a/docs/reference/commands/README.md +++ b/docs/reference/commands/README.md @@ -86,7 +86,8 @@ Full records: [raw-material-and-uploads.md](raw-material-and-uploads.md) - [`raw status`](raw-material-and-uploads.md#raw-status) - Report which captured bundles under `raw/` have fallen behind their repository, with each changed file as `A`/`M`/`D`. - [`raw pending`](raw-material-and-uploads.md#raw-pending) - List what waits in `incoming/`, oldest first, and name the entry an ingest without an argument takes next. - [`raw accept`](raw-material-and-uploads.md#raw-accept) - Promote one or more files, or one folder, from `incoming/` into `raw/`. -- [`upload list`](raw-material-and-uploads.md#upload-list) - List every MCP submission currently waiting in the quarantine (`mcp-upload/`). +- [`upload collect`](raw-material-and-uploads.md#upload-collect) - Pull material from the configured `folder` and `command` intake channels into the quarantine - never past it. +- [`upload list`](raw-material-and-uploads.md#upload-list) - List every submission currently waiting in the quarantine (`quarantine/`), whatever channel it came through. - [`upload show`](raw-material-and-uploads.md#upload-show) - Print one submission's manifest in full. - [`upload accept`](raw-material-and-uploads.md#upload-accept) - **Upload Review Gate:** promote a submission's file from quarantine into `incoming/`. - [`upload reject`](raw-material-and-uploads.md#upload-reject) - Delete a submission's material, keeping only its ledger trail. diff --git a/docs/reference/commands/git.md b/docs/reference/commands/git.md index 1d18c3f..b14926c 100644 --- a/docs/reference/commands/git.md +++ b/docs/reference/commands/git.md @@ -237,7 +237,7 @@ tools/wikitool backup run **Behaviour** - Mirrors `.wikitool-local.json`, `ENVIRONMENT.md` if present, every file in `incoming/` but `.gitkeep`, and the `extra` paths of the `backup` section into `//`, then writes the manifest `.chemenu-backup.json` there: time, stack version, machine name, and per file its relative path, size and sha256. -- Never mirrors `mcp-upload/`, `reports/`, `.wikitool-tools.json`, `tools/.venv/`, the published skills or `.git/` - unless an `extra` path names one of the first ones. The secrets file lies outside the instance and is never read. +- Never mirrors `quarantine/`, `reports/`, `.wikitool-tools.json`, `tools/.venv/`, the published skills or `.git/` - unless an `extra` path names one of the first ones. The secrets file lies outside the instance and is never read. - Writes and deletes only under `//`. Deletes only a file the previous manifest lists and the instance no longer has; a file there the manifest does not list stays as it is. A file whose size and sha256 match the previous manifest is not rewritten. - A symbolic link or junction inside a mirrored directory is skipped and named in the output, never followed. - `publish` runs the same step after every push when `machine_data` is true. diff --git a/docs/reference/commands/instance-health.md b/docs/reference/commands/instance-health.md index 5702e6f..425087b 100644 --- a/docs/reference/commands/instance-health.md +++ b/docs/reference/commands/instance-health.md @@ -38,7 +38,7 @@ tools/wikitool doctor --json - Legacy settings (`legacy-settings`): one `FAIL` for each per-feature settings file of an earlier version (telemetry, remotes, upload, task tracker) that is still there. Nothing reads them any more, so the settings they hold are not in effect; the fix is `tools/wikitool config migrate`. - Synced folder (`sync-folder`): a `WARN` when the git directory, links and junctions resolved, lies under a folder named by `OneDrive`, `OneDriveCommercial` or `OneDriveConsumer` (compared without regard to case on Windows) or under `~/Library/CloudStorage/` (macOS); `OK` otherwise, never a `FAIL`. - Backup (`backup`): the `backup` section absent is `OK`, malformed is a `FAIL`. Configured, a `WARN` when `/git` is not a bare repository, `origin` does not push there, or - with `machine_data` true - no machine-data backup exists yet; otherwise `OK` with the age of the last machine-data backup. -- MCP `submit` tool: whether the `upload` section of the settings is present, absent or malformed, its limits, and how many submissions wait in `mcp-upload/`. Absent is `OK` and means the write path does not exist at all; malformed is a `FAIL`. +- Intake channels (`upload-intake`): the channels of the `intake` section with their handler, and how many submissions wait in `quarantine/`. Absent is `OK` and means no channel - no `submit` tool, nothing to collect; malformed is a `FAIL`, and so is an `mcp-upload/` of an earlier version that still holds submissions (the message names the command that moves it). A `folder` channel whose `path` does not exist is a `WARN`. - Task tracker: the `tasks` section present, absent or malformed - absent is `OK` (no tracker configured), malformed is a `FAIL`. When `WIKITOOL_TASKS_CONFIG` is set, the profile it names is read instead of the `tasks`, `superproductivity` and `caldav` sections and the finding names it; a set variable that names no file is a `FAIL`, never `OK`. - For a configured `superproductivity` provider, the configured `access` path's own state: `access: "api"` reports whether its local REST API answers `GET /health` with a ready renderer right now, `access: "snapshot"` whether a backup file is ready. The other access path is never attempted, and neither state is ever a `FAIL`. - For a configured `caldav` provider, whether the server is reachable and Basic auth succeeds - never a `FAIL`; only a broken config block is. diff --git a/docs/reference/commands/raw-material-and-uploads.md b/docs/reference/commands/raw-material-and-uploads.md index 4d6c2c8..96c9a9d 100644 --- a/docs/reference/commands/raw-material-and-uploads.md +++ b/docs/reference/commands/raw-material-and-uploads.md @@ -12,6 +12,7 @@ Bringing new material into `raw/`: fetching and capturing it, seeing what is wai - [`raw status`](#raw-status) - [`raw pending`](#raw-pending) - [`raw accept`](#raw-accept) +- [`upload collect`](#upload-collect) - [`upload list`](#upload-list) - [`upload show`](#upload-show) - [`upload accept`](#upload-accept) @@ -365,7 +366,7 @@ tools/wikitool raw pending --json **Behaviour** -- A candidate is a top-level entry of `incoming/`: a single `file`, a `bundle` of top-level files sharing a stem (a `raw fetch` pair), or a `folder` with every file below it. Dotfiles, empty directories and their contents are none; `mcp-upload/` is outside `incoming/` and never listed. +- A candidate is a top-level entry of `incoming/`: a single `file`, a `bundle` of top-level files sharing a stem (a `raw fetch` pair), or a `folder` with every file below it. Dotfiles, empty directories and their contents are none; `quarantine/` is outside `incoming/` and never listed. - Order: oldest first by modification time. A bundle or folder counts as new as its newest file; a tie goes by name. The mtime is when a document last changed only if it was copied with its timestamps kept (`cp -p`, `rsync -a`, an unpacked archive) - for a download or a `raw fetch` it is merely when it was dropped. - Each candidate shows its path(s), kind, file count and mtime, and whether `raw accept` would take it as it stands - the same checks, minus `--fidelity`/`--authority`. One it would refuse is listed with the reason and skipped: it needs a human. - The default is the first candidate `raw accept` would take, and the output names it. @@ -495,9 +496,64 @@ tools/wikitool raw accept incoming/chemenu --replaces-bundle raw/2026/10/chemenu - `wikitool types describe source` - the capture field values - `wikitool new source` - the source page for a promoted file +## `upload collect` + +Pull material from the configured `folder` and `command` intake channels into the quarantine - never past it. + +**Synopsis** + +- `wikitool upload collect [...]` + +**Options** + +```text +Arguments: + names... Channels to collect from; all pulled channels when none is named + +Options: + -h, --help Show this message and exit. +``` + +**Examples** + +```bash +tools/wikitool upload collect +tools/wikitool upload collect onedrive +``` + +**Behaviour** + +- Without arguments, collects from every channel of the `intake` section whose handler is `folder` or `command`; with names, from those channels only. A name no channel has, or the name of the `mcp` channel (which is pushed, not pulled), exits 1 before anything happens. +- Everything a channel delivers lands in `quarantine//` unchanged, with a manifest that names `channel`, `handler` and `origin`; the submitter is `unknown`. Nothing reaches `incoming/` or `raw/` - that is `upload accept`, behind its gate, and an agent may run `collect` because collecting gives no trust. +- `folder`: only the top level of `path`. A file is taken once its modification time and, where the platform has it, its creation time are older than `settle_seconds`, its extension and size fit, and its name passes the filename rules. It is removed from the folder only after its copy is fully written and the source still has the size and sha256 that were read; if it changed meanwhile, nothing is kept and it stays. A file whose sha256 already waits in the quarantine is reported as a duplicate and removed from the folder, because its content is kept. Subdirectories, hidden files (names starting with `.` or `~$`, or the hidden or system attribute on Windows), symbolic links, other extensions and oversized files are left untouched and reported. +- `command`: the program of `commands.` runs without a shell, with stdin closed, a 300 s timeout and an empty staging directory under `quarantine/.staging/` as its last argument. On exit 0 each regular file at the top of that directory with an allowed extension and size becomes a submission; on anything else nothing is taken and the end of stderr is printed. The staging directory is always removed. +- Channels are independent: when one fails the others still run, and the exit status is 1 if any channel or file failed. With no pulled channel it prints `nothing to collect` and exits 0. +- Exits 1, before it does anything, while an `mcp-upload/` of an earlier version still holds submissions, and when the `intake` section is malformed. +- Not idempotent: a second run finds the folder emptied. + +**Properties** + +- Changes files: yes +- Idempotent: no +- Atomic: No - per file: one copy into `quarantine//` (manifest last), then one removal of the source; a channel's failure does not stop the others +- Counts toward the session call budget: yes +- Can reach the network: no + +**Exit codes** + +- `0` - success +- `1` - Unknown channel name, an `mcp` channel as argument, a malformed `intake` section, or an `mcp-upload/` that still holds submissions +- `1` - A channel or a file failed (folder missing, program exited non-zero, timed out or could not be started, a source that could not be removed) + +**See also** + +- `wikitool upload list` - what waits for review afterwards +- `wikitool upload accept` - the gate out of the quarantine +- `docs/how-to/collect-from-a-synced-folder.md` + ## `upload list` -List every MCP submission currently waiting in the quarantine (`mcp-upload/`). +List every submission currently waiting in the quarantine (`quarantine/`), whatever channel it came through. **Synopsis** @@ -519,8 +575,9 @@ tools/wikitool upload list --json **Behaviour** -- Lists every submission waiting in `mcp-upload/`, oldest id first: id, filename, size, submitter. -- Only ever non-empty when the `upload` section of `.wikitool-local.json` opts the checkout into the MCP server's `submit` tool. +- Lists every submission waiting in `quarantine/`, oldest id first: id, filename, size, channel, submitter. +- Only ever non-empty when the `intake` section of `.wikitool-local.json` declares a channel. +- Exits 1, before it does anything, while an `mcp-upload/` of an earlier version still holds submissions - the message names the command that moves it. - A submission directory with a corrupt manifest is silently skipped. - Never fails; read-only and safe to retry freely. @@ -568,7 +625,8 @@ tools/wikitool upload show **Behaviour** -- Prints one submission's manifest in full: filename, size, sha256, submitter, submitter source (the header name, not a claim the header was honest), submission time. +- Prints one submission's manifest in full: filename, size, sha256, submitter, submitter source (the header name or the handler, not a claim the header was honest), submission time, and where it came from: channel, handler, origin (the source file's path, `command:` or `mcp:submit`). +- A submitter of `unknown` means the channel has no sender to trust - read channel and origin instead. - What a reviewer reads before `upload accept`. - Read-only. @@ -618,7 +676,7 @@ tools/wikitool upload accept --confirm # re-run after exit 42, o **Behaviour** -- Promotes a submission's file from `mcp-upload//` into `incoming/`, deletes the quarantine directory, and appends an `accepted` event to `mcp-upload/ledger.jsonl`. +- Promotes a submission's file from `quarantine//` into `incoming/`, deletes the quarantine directory, and appends an `accepted` event to `quarantine/ledger.jsonl`. - Upload Review Gate: without a matching `--confirm`, exits 42 and prints the manifest in full plus the exact re-run line - one submission at a time. The gate check runs before anything is moved. - The token digests id, filename, size, sha256 and submitter, so an edited or superseded manifest invalidates it. - An occupied `incoming/` is an ordinary validation error, not the gate. @@ -635,7 +693,7 @@ tools/wikitool upload accept --confirm # re-run after exit 42, o **Exit codes** - `0` - success -- `1` - Unknown or malformed submission id, or the submission's file is missing from `mcp-upload//` +- `1` - Unknown or malformed submission id, or the submission's file is missing from `quarantine//` - `1` - `incoming/` already exists - `42` - Upload Review Gate: `--confirm` is absent or does not match the manifest's current token diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 881584c..571097d 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -27,7 +27,7 @@ something you run. - [Super Productivity](#super-productivity) - [CalDAV](#caldav) - [Profiles](#profiles) -- [MCP submissions](#mcp-submissions) +- [Intake channels](#intake-channels) - [Site handlers](#site-handlers) - [Fetch routes](#fetch-routes) - [Backup to a synced folder](#backup-to-a-synced-folder) @@ -92,7 +92,7 @@ Each file is a JSON object of sections, one per feature, beside two keys of its `instance_id` sits in `wikitool.json` only. The first `wikitool config set` creates the file and the id; the id names the directory of the secrets file below. The sections are [telemetry](#telemetry), [remotes](#allowed-push-urls), [tasks, superproductivity and caldav](#task-tracker) and -[upload](#mcp-submissions), [sites](#site-handlers), [fetch](#fetch-routes) and +[intake](#intake-channels), [sites](#site-handlers), [fetch](#fetch-routes) and [backup](#backup-to-a-synced-folder). `config set` names the sections it knows, and `doctor` reports a section it does not read as a `WARN`. @@ -121,7 +121,7 @@ for a human to unlock. Setting 1Password up: [use-1password-for-settings.md](../ **Never in the committed file:** a field that names a program to run or affects a gate - not even as a reference. `config set` does not write such a field either (exit 42); the user edits `.wikitool-local.json` by hand, and `doctor` checks it afterwards. The fields marked `gate` or -`names a command` in the tables below are these: the publish allowlist, the upload limits and the +`names a command` in the tables below are these: the publish allowlist, the intake channels and the external fetchers. `wikitool doctor` reports one line per tier - `settings-instance`, `settings-checkout`, @@ -371,45 +371,92 @@ several trackers in turn. A profile is checked like the local file, except that value is allowed in it; a secret is a reference there too, never a value. The live suite takes its throwaway tokens from `WIKITOOL_SECRET_`. -## MCP submissions +## Intake channels -The `upload` section of `.wikitool-local.json` in the served corpus turns on the MCP read server's -optional `submit` tool. What the authentication in front of the server has to guarantee before you -create it is in -[Set up the MCP read server](../how-to/set-up-the-mcp-server.md#optional-the-submit-path). Every field -is marked `gate`: the limits are the user's to set by hand, and `config set` exits 42 for them. +The `intake` section of `.wikitool-local.json` declares the ways a document gets into the +quarantine `quarantine/` from somewhere other than this terminal. Each channel has a name and a +handler: - +| Handler | Source | How it arrives | +|---|---|---| +| `mcp` | The MCP read server's optional `submit` tool | Pushed by a remote caller. At most one channel has this handler. | +| `folder` | A directory outside the instance that a sync client fills (OneDrive and the like) | Pulled by `wikitool upload collect`; what is taken is removed from the folder. | +| `command` | A program of this checkout, named in `commands` | Pulled by `wikitool upload collect`; the program leaves its files in an empty staging directory. | + +All of them end in the same place: a submission nobody has looked at, which `wikitool upload accept` +promotes into `incoming/` after a human has cleared the Upload Review Gate. What the authentication +in front of the server has to guarantee for the `mcp` handler is in +[Set up the MCP read server](../how-to/set-up-the-mcp-server.md#optional-the-submit-path); how to +collect from a synced folder or a program is in +[Collect documents from a synced folder](../how-to/collect-from-a-synced-folder.md). Every field is +marked `gate` or `names a command`: the channels are the user's to set by hand, and `config set` +exits 42 for them. + + | Field | Tier | Type | Required | Default | Description | |-------|------|------|----------|---------|-------------| -| `identity_header` | checkout, gate | string | no | `"X-Forwarded-User"` | The request header the authenticating middleware sets to the submitter's identity on every request. An empty value means the default. | -| `max_bytes` | checkout, gate | positive integer | yes | - | Largest single submission, in bytes. A number written as a string is accepted. | -| `allowed_extensions` | checkout, gate | array of strings | yes | - | The file extensions `submit` accepts, each starting with `.`. Compared case-insensitively. Entries are read as strings and lower-cased; duplicates collapse. | -| `quota_submissions_per_day` | checkout, gate | positive integer | yes | - | Submissions one submitter may make over any rolling 24 hours. A number written as a string is accepted. | -| `quota_bytes_per_day` | checkout, gate | positive integer | yes | - | Bytes one submitter may submit over any rolling 24 hours. A number written as a string is accepted. | +| `channels` | checkout, gate | object | yes | - | Channel name (lower-case letters, digits and hyphens) to channel. The name goes into the manifest of what the channel delivers and is the argument of `upload collect`. Every channel has `handler` (`mcp`, `folder` or `command`), `max_bytes` (the largest single file) and `allowed_extensions` (each starting with `.`, compared case-insensitively). Then per handler: `mcp` takes `identity_header` (the request header the authenticating middleware sets to the submitter's identity, default `X-Forwarded-User`) and, required, `quota_submissions_per_day` and `quota_bytes_per_day` (per submitter, over any rolling 24 hours); `folder` takes `path` (required: an absolute path outside the instance, read without descending into subdirectories) and `settle_seconds` (default 60: a file whose modification or creation time is younger is left alone); `command` takes `command` (required: the name of an entry of `commands`). At most one channel has the handler `mcp`. A field the handler does not take is an error. | +| `commands` | checkout, names a command | object | no | `{}` | Command name to command: an array of the program and its arguments, run by the `command` handler without a shell, with stdin closed and a timeout of 300 s, and a staging directory appended as the last argument. Exit 0 turns each regular file at the top level of that directory into a submission; anything else takes nothing over and prints the end of the program's stderr. | -- **Absent:** The `submit` tool is not registered: the write path does not exist. -- **Malformed:** An error. The MCP server does not start and `doctor` reports `FAIL`. A broken section is never read as "no limits". +- **Absent:** No channel: the `submit` tool is not registered and `upload collect` has nothing to collect. A valid state; `doctor` reports `OK`. +- **Malformed:** An error. The MCP server does not start, `upload collect` stops with exit 1 and `doctor` reports `FAIL`. A broken section is never read as "no channel". Example: ```json { - "upload": { - "identity_header": "X-Forwarded-User", - "max_bytes": 10485760, - "allowed_extensions": [ - ".md", - ".pdf" - ], - "quota_submissions_per_day": 20, - "quota_bytes_per_day": 52428800 + "intake": { + "channels": { + "mcp": { + "handler": "mcp", + "identity_header": "X-Forwarded-User", + "max_bytes": 10485760, + "allowed_extensions": [ + ".md", + ".pdf" + ], + "quota_submissions_per_day": 20, + "quota_bytes_per_day": 52428800 + }, + "onedrive": { + "handler": "folder", + "path": "C:\\Users\\\\OneDrive - \\Chemenu-Eingang", + "max_bytes": 52428800, + "allowed_extensions": [ + ".pdf", + ".docx", + ".msg", + ".eml", + ".md" + ], + "settle_seconds": 60 + }, + "scanner": { + "handler": "command", + "command": "scanner-pull", + "max_bytes": 52428800, + "allowed_extensions": [ + ".pdf" + ] + } + }, + "commands": { + "scanner-pull": [ + "pwsh", + "-NoProfile", + "-File", + "C:\\tools\\scanner-pull.ps1" + ] + } } } ``` - + -Size and quota are your deployment's decision, not defaults worth copying blindly. +Size and quota are your deployment's decision, not defaults worth copying blindly. A field that does +not belong to the channel's handler is an error, not ignored: `identity_header` and the `quota_*` +fields belong to `mcp`, `path` and `settle_seconds` to `folder`, `command` to `command`. A `folder` +`path` must be absolute and lie outside the instance, and `upload collect` reads only its top level. ## Site handlers @@ -590,7 +637,7 @@ Until version 10 every feature above had a file of its own at the root: `.wikito `.wikitool-remotes.json`, `.wikitool-upload.json` and `.wikitool-tasks.json`. Nothing reads them any more. A command that needs the setting such a file held stops with exit 42 and names `tools/wikitool config migrate`, `doctor` reports each one as a `FAIL`, and only the telemetry hook -ignores it. `config migrate` moves the content into the sections above, each value into the file its +ignores it. `config migrate` moves the content into the sections above (the upload file becomes the channel `mcp` of `intake`), each value into the file its tier names and each secret into the secrets file, and then removes the old files; it is run once per checkout. `.wikitool-tools.json` is not among them and stays. The migration step for an instance is [10.0.0-settings-tiers.md](../../instructions/migrations/10.0.0-settings-tiers.md). diff --git a/instructions/gates.md b/instructions/gates.md index 9a72b1a..8c6430b 100644 --- a/instructions/gates.md +++ b/instructions/gates.md @@ -117,11 +117,13 @@ it leaves open exactly the window it closes. ### Upload Review Gate -The MCP server's opt-in `submit` tool (the `upload` section of `.wikitool-local.json`) writes into a -quarantine, `mcp-upload/`, that no other command reads. This gate is the only door out of it: +The intake channels (the `intake` section of `.wikitool-local.json`: the MCP server's opt-in +`submit` tool, a synced `folder`, a `command`) write into a quarantine, `quarantine/`, that no other +command reads. This gate is the only door out of it: `wikitool upload accept ` refuses without a matching `--confirm `, printing the -submission's manifest in full - filename, size, sha256, submitter, and `submitter_source` (the -*header* the submitter's name came from, not a verified fact) - plus the exact re-run line. +submission's manifest in full - filename, size, sha256, submitter, `submitter_source` (the +*header* the submitter's name came from, not a verified fact; the handler for a channel with no +sender, whose `submitter` is `unknown`), and `channel`, `handler` and `origin` - plus the exact re-run line. Same shape as the Mass-Update Gate, scoped to one submission instead of a changeset: the token digests id/filename/size/sha256/submitter, so an edited or superseded manifest invalidates it the @@ -132,7 +134,7 @@ Mass-Update Gate's review report versus this file's exit-42 procedure. `wikitool upload reject --reason ""` is the other way out, and it has **no gate at all** - rejecting needs no clearance, only accepting a stranger's file into the pipeline does. It -deletes the material and keeps only the reason and a sha256 in `mcp-upload/ledger.jsonl`. +deletes the material and keeps only the reason and a sha256 in `quarantine/ledger.jsonl`. ### Guideline Push Gate diff --git a/instructions/ingest-queue.md b/instructions/ingest-queue.md index fbc729b..2630d6f 100644 --- a/instructions/ingest-queue.md +++ b/instructions/ingest-queue.md @@ -1,18 +1,19 @@ --- type: types/instruction.md name: ingest-queue -description: Review a document submitted from outside through the MCP submit tool before it is promoted into incoming/ - what to check, how the quarantine and its ledger work, and the Upload Review Gate that stands between a submission and raw/ +description: Review a document that reached the quarantine from outside - through the MCP submit tool, a synced folder or a command of this checkout - before it is promoted into incoming/ - what to check, how the quarantine and its ledger work, and the Upload Review Gate that stands between a submission and raw/ --- # Reviewing an external submission -An `upload` section in `.wikitool-local.json` opts a checkout into a sixth MCP tool, `submit` - -documents pushed by a caller that is not this terminal, into a quarantine no -ordinary command reads. This is the human half of that path: what a reviewer -checks before letting one through, and how `upload accept`/`upload reject` -work. What the tool itself enforces (identity, size, extension, quota, -duplicate-hash) is `chemenu/upload.py`'s job and is not repeated here - read -this when a submission is already waiting and a decision is due. +The `intake` section in `.wikitool-local.json` opts a checkout into one or more *intake channels*: +a sixth MCP tool, `submit` (handler `mcp`), a folder a sync client fills (`folder`) or a program of +this checkout (`command`) - documents that come from somewhere other than this terminal, into a +quarantine, `quarantine/`, that no ordinary command reads. This is the human half of that path: +what a reviewer checks before letting one through, and how `upload accept`/`upload reject` work. +What the channels themselves enforce (identity, size, extension, quota, duplicate-hash, settle +time) is `chemenu/upload.py`'s and `chemenu/collect.py`'s job and is not repeated here - read this +when a submission is already waiting and a decision is due. ## Contents @@ -26,7 +27,8 @@ this when a submission is already waiting and a decision is due. ## When to run -- `wikitool upload list` shows one or more submissions waiting. +- `wikitool upload list` shows one or more submissions waiting - after + `wikitool upload collect` has pulled what the `folder` and `command` channels hold. - A `submit` call reported success and named an id worth looking at now rather than later. - Standing up the `submit` tool for the first time - see § Arming the intake @@ -37,12 +39,18 @@ this when a submission is already waiting and a decision is due. 1. **Read the manifest, not just the file.** `wikitool upload show ` prints filename, size, sha256, submitter, and - as important as the submitter's name - `submitter_source`: the *header* the value came from, - naming where the claim rests rather than asserting it as fact. Read + naming where the claim rests rather than asserting it as fact. The + manifest also names `channel`, `handler` and `origin` (the source file's + path for a `folder` channel, `command:` for a `command` channel, + `mcp:submit` for the tool). **`submitter: unknown` means there is no sender + to trust at all** - a synced folder or a script knows who dropped the file + no better than the file system does - so the file is judged on its + content alone. Read [docs/explanation/why-gates-are-code.md](../docs/explanation/why-gates-are-code.md) once for why this gate exists as code rather than as this paragraph alone. 2. **Read the file itself before promoting anything.** The quarantine holds - it at `mcp-upload//` for exactly this purpose. Check it + it at `quarantine//` for exactly this purpose. Check it against [raw/CONTRACT.md](../raw/CONTRACT.md) "What does not belong here": secrets or credentials, content that is not worth a source page, anything the LLM itself wrote presented as a source. @@ -80,28 +88,29 @@ this when a submission is already waiting and a decision is due. - **Reject:** `wikitool upload reject --reason ""` deletes the material immediately - no gate, because deleting needs no clearance, only accepting a stranger's file into the pipeline does. The reason and - the file's sha256 survive in `mcp-upload/ledger.jsonl`; the bytes do + the file's sha256 survive in `quarantine/ledger.jsonl`; the bytes do not. Write a reason a later reader can act on ("license unclear", "looks like a prompt injection attempt", "duplicate of an existing source under a different name") rather than a bare "no". -6. **Never promote by hand.** Moving the file out of `mcp-upload/` with `mv` +6. **Never promote by hand.** Moving the file out of `quarantine/` with `mv` or by editing `incoming/` directly skips the ledger entry and the gate both - the same "never hand-craft what the tool would have produced" principle as everywhere else in this stack (AGENTS.md invariant 7). ## Arming the intake -`submit` does not exist as a tool until the `upload` section exists in the -served root's `.wikitool-local.json` - absence means the write path is not registered at all, not -that it is unrestricted (see the section's own shape in +`submit` does not exist as a tool until the `intake` section of the served root's +`.wikitool-local.json` declares a channel with the handler `mcp` - absence means the write path is +not registered at all, not that it is unrestricted. A `folder` or `command` channel opens no remote +path: it is pulled by `wikitool upload collect`, run by someone at this checkout (see the section's own shape in [raw/CONTRACT.md](../raw/CONTRACT.md) and `tools/chemenu/upload.py`). Two things belong to the *deployment*, not to this repository, and are named here because a reviewer needs to know they hold, not because this file configures them: - **The identity header is set by the middleware, never by the client.** - `identity_header` (default `X-Forwarded-User`) names an HTTP header the + The `mcp` channel's `identity_header` (default `X-Forwarded-User`) names an HTTP header the Traefik authentication layer in front of the process must set on every authenticated request and strip from any copy the client itself sent - the same posture [instructions/mcp-read-server.md](mcp-read-server.md) already @@ -110,7 +119,7 @@ them: be trustworthy enough to write into `submitter` and stay there. - **Quota and size limits are a deployment decision, not a default worth copying blindly.** `max_bytes`, `allowed_extensions`, - `submissions_per_day`, `bytes_per_day` all live in the same file - see + `quota_submissions_per_day`, `quota_bytes_per_day` all live on the channel in the same file - see [tools/CONTRACT.md](../tools/CONTRACT.md) for the exact shape. ## Decision points diff --git a/instructions/mcp-read-server.md b/instructions/mcp-read-server.md index babf107..5861362 100644 --- a/instructions/mcp-read-server.md +++ b/instructions/mcp-read-server.md @@ -96,7 +96,7 @@ everything an operator needs that is *true of the software* rather than of one i stamped `"commit": null`, because a dirty tree corresponds to no revision. **Never add `git clean` to this sync.** `reset --hard` leaves every gitignored path alone by - design, which is exactly what keeps `mcp-upload/` (the `submit` tool's own quarantine) and + design, which is exactly what keeps `quarantine/` (the intake channels' quarantine) and `reports/telemetry/` intact across a sync - a `git clean -xd` bolted on "to tidy up" would delete a submission nobody has reviewed yet, silently, on the next poll. @@ -112,8 +112,8 @@ everything an operator needs that is *true of the software* rather than of one i - **Asked to expose a write tool?** Five of the six tools have none, structurally: the server imports nothing under `chemenu.commands`, so `new`, `touch`, `xref`, `cite`, `publish` and `migrate` are unreachable from it rather than filtered out of a list. The one exception is - `submit` (opt-in via the `upload` settings section): it may write, but only into - `mcp-upload/`, a quarantine no other command reads - a **positive list** enforced in code + `submit` (opt-in via a channel with the handler `mcp` in the `intake` settings section): it may write, but only into + `quarantine/`, a quarantine no other command reads - a **positive list** enforced in code (`chemenu.upload._write_atomic_within`), not an absence. The commands that move a submission *out* of that quarantine (`upload accept`/`upload reject`) still have the absence property: they live under `chemenu.commands` and stay unreachable from the server. Reviewing what diff --git a/instructions/migrations/10.0.0-settings-tiers.md b/instructions/migrations/10.0.0-settings-tiers.md index 6f2f352..e5cbabf 100644 --- a/instructions/migrations/10.0.0-settings-tiers.md +++ b/instructions/migrations/10.0.0-settings-tiers.md @@ -1,7 +1,7 @@ --- type: types/instruction.md name: 10.0.0-settings-tiers -description: Move the settings of the four per-feature files .wikitool-telemetry.json, .wikitool-remotes.json, .wikitool-upload.json and .wikitool-tasks.json into the tiered settings with `wikitool config migrate` - run once per checkout; without it the Publish-Remote Gate and the upload limits are not in effect and the commands that need them exit 42. +description: Move the settings of the four per-feature files .wikitool-telemetry.json, .wikitool-remotes.json, .wikitool-upload.json and .wikitool-tasks.json into the tiered settings with `wikitool config migrate`, and move the quarantine mcp-upload/ to quarantine/ - run once per checkout; without it the Publish-Remote Gate and the upload limits are not in effect and the commands that need them exit 42. manual: true migrates_to: 10.0.0 migration_kind: mechanical @@ -13,7 +13,8 @@ Until 10.0.0 four features kept their settings in a file of their own at the roo checkout: `.wikitool-telemetry.json`, `.wikitool-remotes.json`, `.wikitool-upload.json` and `.wikitool-tasks.json`. 10.0.0 reads none of them. The same settings are now sections of the [tiered settings](../../docs/reference/configuration.md#tiered-settings) - `telemetry`, -`remotes`, `upload`, and `tasks` with `superproductivity` and `caldav` - and each field sits in +`remotes`, `intake` (the old `upload` file becomes its channel `mcp`), and `tasks` with +`superproductivity` and `caldav` - and each field sits in the file its tier names: `wikitool.json` (committed) for what is the same in every clone, `.wikitool-local.json` for what is true of one checkout, and for a token or a password the secrets file, with only a reference in `wikitool.json`. @@ -23,6 +24,13 @@ Publish-Remote Gate, no upload limits, no tracker. `wikitool config migrate` mov removes the old files. It is `mechanical` because nothing is decided per file: every setting keeps its effective value, defaults included. +10.0.0 also renames the quarantine of the intake channels from `mcp-upload/` to `quarantine/`, +because it no longer holds only what the MCP `submit` tool received. A checkout whose `mcp-upload/` +still holds submissions that wait for review moves that directory once, by hand - nothing moves it +automatically, since material that waits for a decision is not the tool's to relocate. Until it +is moved, `doctor` reports a `FAIL` on `upload-intake` with the command, the `upload` commands +exit 1 naming it, and the MCP server does not start. + ## Contents @@ -52,8 +60,9 @@ command below. tools/wikitool doctor ``` - Each old file that is still present is one `legacy-settings` line. None, and the checkout has - nothing to move: go to step 4. + Each old file that is still present is one `legacy-settings` line, and an `mcp-upload/` that + still holds submissions is a `FAIL` on the `upload-intake` line. Neither, and the checkout has + nothing to move: go to step 5. 2. **Move it:** @@ -67,18 +76,33 @@ command below. removes the old ones; if anything fails, nothing is changed and the old files are as they were. A second run says there is nothing to move. -3. **Check it:** +3. **Move the quarantine, if `mcp-upload/` still holds submissions.** In the instance root: + + ```bash + mv mcp-upload quarantine # POSIX shells + Move-Item mcp-upload quarantine # PowerShell + ``` + + If `quarantine/` exists already (the new version created it), move the entries of + `mcp-upload/` into it one by one - each is a directory named by its id, plus `ledger.jsonl`, + which is appended to the existing ledger rather than replacing it - and remove the empty + `mcp-upload/` afterwards. Open submissions keep their ids and are listed by `upload list` again. + The root `.gitignore` of an instance carries `/quarantine/` from this version on; `dist upgrade` + brings the entry, and the quarantine writes its own `.gitignore` as well. + +4. **Check it:** ```bash tools/wikitool doctor git status --short ``` - `doctor` shows no `legacy-settings` line, and the settings lines are `OK`. `git status` shows + `doctor` shows no `legacy-settings` line, the `upload-intake` line is not a `FAIL`, and the + settings lines are `OK`. `git status` shows `wikitool.json` as new or changed: it holds the instance-tier values and the references, and it is committed with the next `publish`. `.wikitool-local.json` does not appear, being ignored. -4. **Record it, once per instance** - not once per checkout: +5. **Record it, once per instance** - not once per checkout: ```bash tools/wikitool migrate done 10.0.0 --pages 0 @@ -89,8 +113,9 @@ command below. ## How to tell a migrated checkout from an unmigrated one -An unmigrated checkout has at least one of the four files at its root, and `doctor` names each. -A migrated one has none; `.wikitool-tools.json` is not among them and stays. +An unmigrated checkout has at least one of the four files at its root, or an `mcp-upload/` that +still holds submissions; `doctor` names each. A migrated one has none; `.wikitool-tools.json` is +not among them and stays. ## Decision points @@ -103,7 +128,8 @@ A migrated one has none; `.wikitool-tools.json` is not among them and stays. carried over. - **The old Publish-Remote allowlist or upload limits go where?** Into `.wikitool-local.json`: both affect a gate, so `config set` does not write them, and the migration is the one command - that does. Later changes to them are the user's own edit. + that does. The upload limits become the channel `mcp` of the `intake` section. Later changes to + them are the user's own edit. - **A profile file `WIKITOOL_TASKS_CONFIG` points at still holds a plain token?** It is not touched by `config migrate`, and it is no longer read as the old format: a profile has the layout of `.wikitool-local.json` and holds references only @@ -112,6 +138,6 @@ A migrated one has none; `.wikitool-tools.json` is not among them and stays. ## Scope -The four files named above, in the checkout the command runs in. `.wikitool-tools.json` and the +The four files named above and the `mcp-upload/` directory, in the checkout the command runs in. `.wikitool-tools.json` and the two tiered settings files are not old files. The machinery - the readers, `config migrate`, `doctor`'s check - arrived with `dist upgrade` and needs nothing from this document. diff --git a/instructions/wiki-ingest/SKILL.md b/instructions/wiki-ingest/SKILL.md index 13bd15f..6712943 100644 --- a/instructions/wiki-ingest/SKILL.md +++ b/instructions/wiki-ingest/SKILL.md @@ -48,12 +48,27 @@ validator complains - and the ticked list is the only record that they happened. `tools/wikitool raw derive ` first and read the `.md` - its output names what to check. Any other binary or image: note its presence and what it shows. - **The user named nothing** ("ingest", "process the inbox")? Pick the entry from the queue: + **The user named nothing** ("ingest", "process the inbox")? First pull what the configured + intake channels hold, then look at what waits for a reviewer, then pick the entry from the queue: ```bash + tools/wikitool upload collect + tools/wikitool upload list tools/wikitool raw pending ``` + `upload collect` takes what the `folder` and `command` channels of the `intake` settings + section hold into the quarantine (`quarantine/`) and prints, per channel, what it took and what + it left with the reason. With no pull channel it says so ("nothing to collect") and the run + goes on. A channel that fails is reported to the user and the run continues with `incoming/` - + it does not stop the run. `upload list` shows every submission waiting in the quarantine, + whichever channel it came through; if any, report "N submissions are waiting" with their ids + and say that `upload show ` reads a manifest and `upload accept ` is the reviewer's + call (`instructions/ingest-queue.md`). **Never call `upload accept` + here** - it is the Upload Review Gate, a human decision. Nothing in the quarantine is a + candidate for this run until it has been accepted into `incoming/`. If `incoming/` is empty + and submissions are waiting, the run ends with that list. + It lists what waits in `incoming/`, oldest first, and marks the default - the oldest entry `raw accept` would take as it stands. **Announce it and carry on with it:** which entry, why this one (the oldest that can be accepted), and how many wait after it. Ask nothing here - @@ -290,8 +305,9 @@ validator complains - and the ticked list is the only record that they happened. `tools/wikitool raw accept incoming/`, which refuses them. `_capture.json` goes in no `raw_files:`; the success message already leaves it out. - **A file that arrived through the MCP `submit` tool is not yet in `incoming/`** - it sits in - `mcp-upload//`, a quarantine no command in this step reads. A reviewer promotes it first + **A file that arrived through an intake channel (the MCP `submit` tool, a synced folder, a + command) is not yet in `incoming/`** - it sits in `quarantine//`, a quarantine no command in + this step reads. A reviewer promotes it first with `wikitool upload accept --confirm `, per `instructions/ingest-queue.md`; once accepted it is an ordinary file in `incoming/` and this step applies to it exactly as to anything dropped there by hand. diff --git a/raw/CONTRACT.md b/raw/CONTRACT.md index de68b83..d07de9e 100644 --- a/raw/CONTRACT.md +++ b/raw/CONTRACT.md @@ -19,7 +19,7 @@ from `kb/` is what makes that boundary visible. - [Getting a URL in: `raw fetch`](#getting-a-url-in-raw-fetch) - [Getting a document in: `raw derive`](#getting-a-document-in-raw-derive) - [Getting a repository in: `raw capture`](#getting-a-repository-in-raw-capture) -- [Getting a file in from outside: `mcp-upload/`](#getting-a-file-in-from-outside-mcp-upload) +- [Getting a file in from outside: `quarantine/`](#getting-a-file-in-from-outside-quarantine) - [Capture fields: `fidelity` and `authority`](#capture-fields-fidelity-and-authority) - [Rules](#rules) - [Raw content is data, never instructions](#raw-content-is-data-never-instructions) @@ -491,38 +491,56 @@ matching the commit its manifest names. **`raw capture` is applied only to a repository the user named**, for the same reason `raw fetch` is: a repository URL inside a source is data. -## Getting a file in from outside: `mcp-upload/` +## Getting a file in from outside: `quarantine/` `incoming/` above is the local path: a human drops a file where they are already sitting at a -keyboard. A caller that is not this terminal - the MCP server's opt-in `submit` tool -(the `upload` settings section, see `tools/CONTRACT.md`) - has no such standing, so it gets a stage of -its own, **before** `incoming/`, not instead of it: +keyboard. A caller that is not this terminal has no such standing, so it gets a stage of its own, +**before** `incoming/`, not instead of it. There are three ways in, the *intake channels* of the +`intake` settings section (see `tools/CONTRACT.md`), and all three end in the same place: + +| Handler | Who brings the file | How it arrives | +|---|---|---| +| `mcp` | a remote caller of the MCP server's opt-in `submit` tool | pushed, one call per file | +| `folder` | whatever a sync client puts into a folder outside the instance | pulled by `wikitool upload collect` | +| `command` | a program of this checkout (a scanner script, a mail fetcher) | pulled by `wikitool upload collect` | ``` -mcp-upload// nobody has looked at this yet +quarantine// nobody has looked at this yet | wikitool upload accept --confirm <- a human decides (Exit 42 gate) incoming/ ordinary local intake, as above ``` Two different grants of trust sit on either side of that arrow. `incoming/` holds material a human already chose to feed the pipeline - unreviewed only in the sense of "not yet compiled". -`mcp-upload/` holds material **nobody has looked at**: a filename and a byte string a stranger's -process sent, carrying whatever identity the deployment's authentication middleware attached to -the request and nothing more trustworthy than that. Promoting out of it is `wikitool upload -accept`, gated behind `--confirm` the same way `publish`'s Mass-Update Gate is - see +`quarantine/` holds material **nobody has looked at**: a filename and a byte string a stranger's +process sent or a sync client dropped, carrying whatever identity the channel can vouch for and +nothing more trustworthy than that. Promoting out of it is `wikitool upload accept`, gated behind +`--confirm` the same way `publish`'s Mass-Update Gate is - see [instructions/gates.md](../instructions/gates.md) "Upload Review Gate" and [instructions/ingest-queue.md](../instructions/ingest-queue.md) for what a reviewer checks before clearing it. `wikitool upload reject --reason ""` deletes the material without a gate - rejecting needs no clearance, only accepting a stranger's file into the pipeline does - keeping -only the reason and a sha256 in `mcp-upload/ledger.jsonl`. +only the reason and a sha256 in `quarantine/ledger.jsonl`. -`mcp-upload/` is stricter than `incoming/` in exactly the way that matters here: no command in +`quarantine/` is stricter than `incoming/` in exactly the way that matters here: no command in the ordinary pipeline reads it at all, not even to report it as a finding, and the identity -attached to a submission is a header value the deployment's middleware set - a claim, not a -verified fact, recorded as such (`submitter_source` names the header it came from). Everything -this file says about `incoming/` and `raw/` - immutability, untrusted content, coverage - -applies unchanged to whatever a submission becomes once a human has accepted it; nothing about -having arrived this way survives the promotion. +attached to a submission is a claim, not a verified fact, recorded as such. A submission through +`mcp` carries the header value the deployment's middleware set (`submitter_source` names the +header); one through `folder` or `command` has no sender at all - `submitter` is `unknown` and +`submitter_source` names the handler - and the manifest's `channel`, `handler` and `origin` say +where it came from. Everything this file says about `incoming/` and `raw/` - immutability, +untrusted content, coverage - applies unchanged to whatever a submission becomes once a human has +accepted it; nothing about having arrived this way survives the promotion. + +**A pulled channel only hands files over.** `upload collect` copies a file into the quarantine +unchanged and removes the source only after the copy is complete and the source still has the +size and sha256 that were read. It reads only the top level of a `folder`'s directory, never +follows a symbolic link, deletes no directory, and leaves whatever it does not take where it is, +with the reason printed. Converting or deriving stays with `raw derive`, after acceptance. + +An instance that ran an earlier version holds its waiting submissions in `mcp-upload/`; no +command reads that directory any more, and the migration guide names the one move that brings +them over. There is deliberately no MCP counterpart to `raw fetch`. A server that fetches any URL a remote caller names fetches it from inside the deployment's network, on that caller's behalf - a diff --git a/tools/CONTRACT.md b/tools/CONTRACT.md index e3062ce..565226b 100644 --- a/tools/CONTRACT.md +++ b/tools/CONTRACT.md @@ -130,7 +130,8 @@ raw capture write non-idempotent budget:counted exit:0,1 raw status read idempotent budget:counted exit:0 Report which captured bundles under `raw/` have fallen behind their repository, with each changed file as `A`/`M`/`D`. raw pending read idempotent budget:counted exit:0 List what waits in `incoming/`, oldest first, and name the entry an ingest without an argument takes next. raw accept write non-idempotent budget:counted exit:0,1 Promote one or more files, or one folder, from `incoming/` into `raw/`. -upload list read idempotent budget:counted exit:0 List every MCP submission currently waiting in the quarantine (`mcp-upload/`). +upload collect write non-idempotent budget:counted exit:0,1 Pull material from the configured `folder` and `command` intake channels into the quarantine - never past it. +upload list read idempotent budget:counted exit:0 List every submission currently waiting in the quarantine (`quarantine/`), whatever channel it came through. upload show read idempotent budget:counted exit:0,1 Print one submission's manifest in full. upload accept write non-idempotent budget:counted exit:0,1,42 **Upload Review Gate:** promote a submission's file from quarantine into `incoming/`. upload reject write non-idempotent budget:counted exit:0,1 Delete a submission's material, keeping only its ledger trail. @@ -1759,7 +1760,7 @@ List what waits in `incoming/`, oldest first, and name the entry an ingest witho **NOTES** -- A candidate is a top-level entry of `incoming/`: a single `file`, a `bundle` of top-level files sharing a stem (a `raw fetch` pair), or a `folder` with every file below it. Dotfiles, empty directories and their contents are none; `mcp-upload/` is outside `incoming/` and never listed. +- A candidate is a top-level entry of `incoming/`: a single `file`, a `bundle` of top-level files sharing a stem (a `raw fetch` pair), or a `folder` with every file below it. Dotfiles, empty directories and their contents are none; `quarantine/` is outside `incoming/` and never listed. - Order: oldest first by modification time. A bundle or folder counts as new as its newest file; a tie goes by name. The mtime is when a document last changed only if it was copied with its timestamps kept (`cp -p`, `rsync -a`, an unpacked archive) - for a download or a `raw fetch` it is merely when it was dropped. - Each candidate shows its path(s), kind, file count and mtime, and whether `raw accept` would take it as it stands - the same checks, minus `--fidelity`/`--authority`. One it would refuse is listed with the reason and skipped: it needs a human. - The default is the first candidate `raw accept` would take, and the output names it. @@ -1865,9 +1866,61 @@ Promote one or more files, or one folder, from `incoming/` into `raw/`. - `wikitool types describe source` - the capture field values - `wikitool new source` - the source page for a promoted file +#### `upload collect` + +Pull material from the configured `folder` and `command` intake channels into the quarantine - never past it. + +**SYNOPSIS** + +- `wikitool upload collect [...]` + +**PROPERTIES** + +- effect: write +- idempotent: no +- atomic: No - per file: one copy into `quarantine//` (manifest last), then one removal of the source; a channel's failure does not stop the others +- budget: counted +- network: no + +**EXAMPLES** + +- `tools/wikitool upload collect` +- `tools/wikitool upload collect onedrive` + +**EXIT STATUS** + +- 0 success +- 1 Unknown channel name, an `mcp` channel as argument, a malformed `intake` section, or an `mcp-upload/` that still holds submissions +- 1 A channel or a file failed (folder missing, program exited non-zero, timed out or could not be started, a source that could not be removed) + +**ON FAILURE** + +- Unknown channel name, an `mcp` channel as argument, a malformed `intake` section, or an `mcp-upload/` that still holds submissions -> Fix the named cause and retry once +- A channel or a file failed (folder missing, program exited non-zero, timed out or could not be started, a source that could not be removed) -> Not fixed by retrying unchanged - read the per-channel lines, fix the channel's folder or program, then run `collect ` for that channel only + +**NEVER** + +- Never follow `collect` with `upload accept` and a `--confirm` token the user has not seen and approved. + +**NOTES** + +- Without arguments, collects from every channel of the `intake` section whose handler is `folder` or `command`; with names, from those channels only. A name no channel has, or the name of the `mcp` channel (which is pushed, not pulled), exits 1 before anything happens. +- Everything a channel delivers lands in `quarantine//` unchanged, with a manifest that names `channel`, `handler` and `origin`; the submitter is `unknown`. Nothing reaches `incoming/` or `raw/` - that is `upload accept`, behind its gate, and an agent may run `collect` because collecting gives no trust. +- `folder`: only the top level of `path`. A file is taken once its modification time and, where the platform has it, its creation time are older than `settle_seconds`, its extension and size fit, and its name passes the filename rules. It is removed from the folder only after its copy is fully written and the source still has the size and sha256 that were read; if it changed meanwhile, nothing is kept and it stays. A file whose sha256 already waits in the quarantine is reported as a duplicate and removed from the folder, because its content is kept. Subdirectories, hidden files (names starting with `.` or `~$`, or the hidden or system attribute on Windows), symbolic links, other extensions and oversized files are left untouched and reported. +- `command`: the program of `commands.` runs without a shell, with stdin closed, a 300 s timeout and an empty staging directory under `quarantine/.staging/` as its last argument. On exit 0 each regular file at the top of that directory with an allowed extension and size becomes a submission; on anything else nothing is taken and the end of stderr is printed. The staging directory is always removed. +- Channels are independent: when one fails the others still run, and the exit status is 1 if any channel or file failed. With no pulled channel it prints `nothing to collect` and exits 0. +- Exits 1, before it does anything, while an `mcp-upload/` of an earlier version still holds submissions, and when the `intake` section is malformed. +- Not idempotent: a second run finds the folder emptied. + +**SEE ALSO** + +- `wikitool upload list` - what waits for review afterwards +- `wikitool upload accept` - the gate out of the quarantine +- `docs/how-to/collect-from-a-synced-folder.md` + #### `upload list` -List every MCP submission currently waiting in the quarantine (`mcp-upload/`). +List every submission currently waiting in the quarantine (`quarantine/`), whatever channel it came through. **SYNOPSIS** @@ -1892,8 +1945,9 @@ List every MCP submission currently waiting in the quarantine (`mcp-upload/`). **NOTES** -- Lists every submission waiting in `mcp-upload/`, oldest id first: id, filename, size, submitter. -- Only ever non-empty when the `upload` section of `.wikitool-local.json` opts the checkout into the MCP server's `submit` tool. +- Lists every submission waiting in `quarantine/`, oldest id first: id, filename, size, channel, submitter. +- Only ever non-empty when the `intake` section of `.wikitool-local.json` declares a channel. +- Exits 1, before it does anything, while an `mcp-upload/` of an earlier version still holds submissions - the message names the command that moves it. - A submission directory with a corrupt manifest is silently skipped. - Never fails; read-only and safe to retry freely. @@ -1933,7 +1987,8 @@ Print one submission's manifest in full. **NOTES** -- Prints one submission's manifest in full: filename, size, sha256, submitter, submitter source (the header name, not a claim the header was honest), submission time. +- Prints one submission's manifest in full: filename, size, sha256, submitter, submitter source (the header name or the handler, not a claim the header was honest), submission time, and where it came from: channel, handler, origin (the source file's path, `command:` or `mcp:submit`). +- A submitter of `unknown` means the channel has no sender to trust - read channel and origin instead. - What a reviewer reads before `upload accept`. - Read-only. @@ -1967,13 +2022,13 @@ Print one submission's manifest in full. **EXIT STATUS** - 0 success -- 1 Unknown or malformed submission id, or the submission's file is missing from `mcp-upload//` +- 1 Unknown or malformed submission id, or the submission's file is missing from `quarantine//` - 1 `incoming/` already exists - 42 Upload Review Gate: `--confirm` is absent or does not match the manifest's current token **ON FAILURE** -- Unknown or malformed submission id, or the submission's file is missing from `mcp-upload//` -> Fix the named argument and retry once +- Unknown or malformed submission id, or the submission's file is missing from `quarantine//` -> Fix the named argument and retry once - `incoming/` already exists -> Not fixed by retrying unchanged - rename or clear it first - Upload Review Gate: `--confirm` is absent or does not match the manifest's current token -> Show the user the command's full output verbatim and stop. Once they have approved it, run the re-run line the output prints, which carries `--confirm `. Without that token, or with a wrong, invented or superseded one, it exits 42 again with the current state @@ -1983,7 +2038,7 @@ Print one submission's manifest in full. **NOTES** -- Promotes a submission's file from `mcp-upload//` into `incoming/`, deletes the quarantine directory, and appends an `accepted` event to `mcp-upload/ledger.jsonl`. +- Promotes a submission's file from `quarantine//` into `incoming/`, deletes the quarantine directory, and appends an `accepted` event to `quarantine/ledger.jsonl`. - Upload Review Gate: without a matching `--confirm`, exits 42 and prints the manifest in full plus the exact re-run line - one submission at a time. The gate check runs before anything is moved. - The token digests id, filename, size, sha256 and submitter, so an edited or superseded manifest invalidates it. - An occupied `incoming/` is an ordinary validation error, not the gate. @@ -2271,7 +2326,7 @@ Mirror this machine's files that are not in git into `//`. **NOTES** - Mirrors `.wikitool-local.json`, `ENVIRONMENT.md` if present, every file in `incoming/` but `.gitkeep`, and the `extra` paths of the `backup` section into `//`, then writes the manifest `.chemenu-backup.json` there: time, stack version, machine name, and per file its relative path, size and sha256. -- Never mirrors `mcp-upload/`, `reports/`, `.wikitool-tools.json`, `tools/.venv/`, the published skills or `.git/` - unless an `extra` path names one of the first ones. The secrets file lies outside the instance and is never read. +- Never mirrors `quarantine/`, `reports/`, `.wikitool-tools.json`, `tools/.venv/`, the published skills or `.git/` - unless an `extra` path names one of the first ones. The secrets file lies outside the instance and is never read. - Writes and deletes only under `//`. Deletes only a file the previous manifest lists and the instance no longer has; a file there the manifest does not list stays as it is. A file whose size and sha256 match the previous manifest is not rewritten. - A symbolic link or junction inside a mirrored directory is skipped and named in the output, never followed. - `publish` runs the same step after every push when `machine_data` is true. @@ -4018,7 +4073,7 @@ Check that this instance is correctly configured. - Legacy settings (`legacy-settings`): one `FAIL` for each per-feature settings file of an earlier version (telemetry, remotes, upload, task tracker) that is still there. Nothing reads them any more, so the settings they hold are not in effect; the fix is `tools/wikitool config migrate`. - Synced folder (`sync-folder`): a `WARN` when the git directory, links and junctions resolved, lies under a folder named by `OneDrive`, `OneDriveCommercial` or `OneDriveConsumer` (compared without regard to case on Windows) or under `~/Library/CloudStorage/` (macOS); `OK` otherwise, never a `FAIL`. - Backup (`backup`): the `backup` section absent is `OK`, malformed is a `FAIL`. Configured, a `WARN` when `/git` is not a bare repository, `origin` does not push there, or - with `machine_data` true - no machine-data backup exists yet; otherwise `OK` with the age of the last machine-data backup. -- MCP `submit` tool: whether the `upload` section of the settings is present, absent or malformed, its limits, and how many submissions wait in `mcp-upload/`. Absent is `OK` and means the write path does not exist at all; malformed is a `FAIL`. +- Intake channels (`upload-intake`): the channels of the `intake` section with their handler, and how many submissions wait in `quarantine/`. Absent is `OK` and means no channel - no `submit` tool, nothing to collect; malformed is a `FAIL`, and so is an `mcp-upload/` of an earlier version that still holds submissions (the message names the command that moves it). A `folder` channel whose `path` does not exist is a `WARN`. - Task tracker: the `tasks` section present, absent or malformed - absent is `OK` (no tracker configured), malformed is a `FAIL`. When `WIKITOOL_TASKS_CONFIG` is set, the profile it names is read instead of the `tasks`, `superproductivity` and `caldav` sections and the finding names it; a set variable that names no file is a `FAIL`, never `OK`. - For a configured `superproductivity` provider, the configured `access` path's own state: `access: "api"` reports whether its local REST API answers `GET /health` with a ready renderer right now, `access: "snapshot"` whether a backup file is ready. The other access path is never attempted, and neither state is ever a `FAIL`. - For a configured `caldav` provider, whether the server is reachable and Basic auth succeeds - never a `FAIL`; only a broken config block is. diff --git a/tools/README.md b/tools/README.md index fb7ea16..9e2712e 100644 --- a/tools/README.md +++ b/tools/README.md @@ -99,7 +99,7 @@ tools/ cli.py Typer app: registers every command, runs the budget gate, renders `-h`/`--help` from cli_contract cli_contract.py one data record per command (name, synopsis, properties, exit status) - the source `-h`, the index, CONTRACT.md's generated region and the human reference under docs/reference/ render from config.py repo layout: root resolution and every path under it - config_shapes.py one record per hand-written .wikitool-*.json (telemetry, remotes, upload, tasks): what its loader accepts, its defaults and `Expected:` text, and what docs/reference/configuration.md renders; also the sections of the tiered settings, each field with its tier; stdlib-only, since the telemetry policy reads it + config_shapes.py one record per hand-written .wikitool-*.json (telemetry, remotes, intake, tasks): what its loader accepts, its defaults and `Expected:` text, and what docs/reference/configuration.md renders; also the sections of the tiered settings, each field with its tier; stdlib-only, since the telemetry policy reads it settings.py the tiered settings: wikitool.json (instance), .wikitool-local.json (checkout), secrets by reference - placement checked per field, references resolved on access through WIKITOOL_SECRET_, the secrets file or `op read` api.py the in-process entry point - point Chemenu at a corpus and read it errors.py ChemenuError / ValidationError / BackendError diff --git a/tools/chemenu/cli_contract.py b/tools/chemenu/cli_contract.py index 31540f2..93f5b12 100644 --- a/tools/chemenu/cli_contract.py +++ b/tools/chemenu/cli_contract.py @@ -288,7 +288,7 @@ GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = ( )), ("Raw material and uploads", ( "raw fetch", "raw login", "raw derive", "raw capture", "raw status", "raw pending", "raw accept", - "upload list", "upload show", "upload accept", "upload reject", + "upload collect", "upload list", "upload show", "upload accept", "upload reject", )), ("Git", ( "sync", "publish", "backup init", "backup run", diff --git a/tools/chemenu/collect.py b/tools/chemenu/collect.py new file mode 100644 index 0000000..660f428 --- /dev/null +++ b/tools/chemenu/collect.py @@ -0,0 +1,268 @@ +"""Pulling material into the quarantine from the channels that are not pushed +(Gitea #213): a folder a sync client fills (`folder`), a program of this +checkout (`command`). + +A handler is only a source. It hands over files unchanged; converting or +deriving stays with `raw derive`, after a human has accepted the submission. +Everything it takes ends in `chemenu.upload.store`, so nothing reaches +`incoming/` or `raw/` any other way than `upload accept`. + +This module is deliberately not part of `chemenu.upload`: it reads outside the +quarantine, deletes in the source folder and starts programs, none of which the +MCP server process may do. Like `upload accept`, it is not importable from there. + +What a pull guarantees: + +- A source file is removed only after its copy in the quarantine is fully + written and the source still has the size and sha256 that were read. +- Outside the quarantine it changes one thing: it removes files it took, directly + in the channel's `path`. It writes nothing there, deletes no directory and + follows no symbolic link. +- Whatever it does not take stays where it is and is reported with its reason. + +No `typer`, no `rich`: the CLI renders a `ChannelResult`. +""" +from __future__ import annotations + +import hashlib +import os +import stat +import subprocess +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Optional + +from chemenu import upload +from chemenu.errors import ValidationError + +COMMAND_TIMEOUT_SECONDS = 300 +_STDERR_TAIL_CHARS = 2000 +_HIDDEN_PREFIXES = (".", "~$") +_WINDOWS_HIDDEN = getattr(stat, "FILE_ATTRIBUTE_HIDDEN", 0) | getattr(stat, "FILE_ATTRIBUTE_SYSTEM", 0) + + +@dataclass +class ChannelResult: + """What one channel's pull did. `taken` and `duplicates` are + `(name, submission id)`, `skipped` is `(name, reason)`, `failed` is + `(name, reason)` for a file that should have been taken and was not. + `error` is set when the channel as a whole could not be pulled.""" + + channel: str + handler: str + taken: list = field(default_factory=list) + duplicates: list = field(default_factory=list) + skipped: list = field(default_factory=list) + failed: list = field(default_factory=list) + error: Optional[str] = None + + @property + def ok(self) -> bool: + return self.error is None and not self.failed + + +def _tail(stderr: bytes) -> str: + text = stderr.decode("utf-8", errors="replace").strip() + if len(text) > _STDERR_TAIL_CHARS: + text = "..." + text[-_STDERR_TAIL_CHARS:] + return text or "(nothing on stderr)" + + +def _digest(path: Path) -> tuple[int, str]: + sha = hashlib.sha256() + size = 0 + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + sha.update(block) + size += len(block) + return size, sha.hexdigest() + + +def source_matches(path: Path, size: int, digest: str) -> bool: + """Whether `path` still is the file that was read: a regular file of the + same size and sha256. The check that gates removing a source.""" + try: + if path.is_symlink() or not path.is_file(): + return False + return _digest(path) == (size, digest) + except OSError: + return False + + +def _hidden(name: str, mode_attributes: int) -> bool: + return name.startswith(_HIDDEN_PREFIXES) or bool(mode_attributes & _WINDOWS_HIDDEN) + + +def _moving(info: os.stat_result, settle_seconds: int, now: float) -> bool: + """Whether a file may still be arriving: its modification time or, where + the platform has it, its creation time is younger than `settle_seconds`.""" + stamps = [info.st_mtime] + born = getattr(info, "st_birthtime", None) + if born is not None: + stamps.append(born) + return any(now - stamp < settle_seconds for stamp in stamps) + + +def _take( + root: Path, channel: upload.Channel, result: ChannelResult, path: Path, name: str, origin: str, + *, after: Optional[Callable[[Path], None]] = None, guard_source: bool, +) -> None: + """Read `path`, store it, and call `after(path)` once it is safe in the + quarantine. Reports into `result`; never raises for a problem with this + one file.""" + try: + before = path.stat() + if before.st_size > channel.max_bytes: + result.skipped.append((name, f"larger than max_bytes ({channel.max_bytes})")) + return + data = path.read_bytes() + except OSError as exc: + result.failed.append((name, f"could not be read ({exc})")) + return + if len(data) != before.st_size: + result.skipped.append((name, "changed while it was read - left for the next run")) + return + if not data: + result.skipped.append((name, "empty")) + return + digest = hashlib.sha256(data).hexdigest() + waiting = upload.digest_to_pending_id(root, digest) + if waiting is not None: + if after is not None and not source_matches(path, len(data), digest): + result.skipped.append((name, "changed while it was read - left for the next run")) + return + result.duplicates.append((name, waiting)) + _finish(result, path, name, after) + return + verify = (lambda: source_matches(path, len(data), digest)) if guard_source else None + try: + manifest = upload.store( + root, channel, filename=name, data=data, + submitter=upload.UNKNOWN_SUBMITTER, submitter_source=channel.handler, + origin=origin, verify=verify, + ) + except upload.SourceChanged: + result.skipped.append((name, "changed while it was copied - nothing was kept, left for the next run")) + return + except (ValidationError, OSError) as exc: + result.failed.append((name, str(exc))) + return + result.taken.append((name, manifest["id"])) + _finish(result, path, name, after, manifest["id"]) + + +def _finish(result: ChannelResult, path: Path, name: str, after, submission_id: str = "") -> None: + if after is None: + return + try: + after(path) + except OSError as exc: + kept = f" (stored as {submission_id})" if submission_id else "" + result.failed.append((name, f"stays in the source, which could not be emptied{kept}: {exc}")) + + +def collect_folder(root: "Path | str", channel: upload.Channel, now: Optional[float] = None) -> ChannelResult: + """Pull the top level of the channel's `path` into the quarantine.""" + root = Path(root) + result = ChannelResult(channel.name, channel.handler) + folder = channel.path + if folder is None or not folder.is_dir(): + result.error = f"the folder {folder} does not exist or is not a directory" + return result + clock = time.time() if now is None else now + try: + entries = sorted(os.scandir(folder), key=lambda entry: entry.name) + except OSError as exc: + result.error = f"the folder {folder} could not be read ({exc})" + return result + for entry in entries: + name = entry.name + path = folder / name + try: + info = entry.stat(follow_symlinks=False) + except OSError as exc: + result.failed.append((name, f"could not be inspected ({exc})")) + continue + if entry.is_symlink(): + result.skipped.append((name, "symbolic link")) + elif stat.S_ISDIR(info.st_mode): + result.skipped.append((name, "subdirectory")) + elif _hidden(name, getattr(info, "st_file_attributes", 0)): + result.skipped.append((name, "hidden")) + elif not stat.S_ISREG(info.st_mode): + result.skipped.append((name, "not a regular file")) + elif Path(name).suffix.lower() not in channel.allowed_extensions: + result.skipped.append((name, "extension not allowed")) + elif info.st_size > channel.max_bytes: + result.skipped.append((name, f"larger than max_bytes ({channel.max_bytes})")) + elif _moving(info, channel.settle_seconds, clock): + result.skipped.append((name, f"still in motion (younger than {channel.settle_seconds} s)")) + else: + try: + upload.sanitize_filename(name) + except ValidationError as exc: + result.skipped.append((name, f"name refused ({exc})")) + continue + _take(root, channel, result, path, name, str(path), after=lambda p: p.unlink(), guard_source=True) + return result + + +def collect_command(root: "Path | str", channel: upload.Channel) -> ChannelResult: + """Run the channel's program with an empty staging directory as its last + argument; take each regular file it left at the top of it.""" + root = Path(root) + result = ChannelResult(channel.name, channel.handler) + origin = f"command:{channel.command_name}" + staging = upload.staging_dir(root) + try: + where = f"the command {channel.command_name!r}" + try: + ran = subprocess.run( + [*channel.command, str(staging)], capture_output=True, stdin=subprocess.DEVNULL, + timeout=COMMAND_TIMEOUT_SECONDS, check=False, + ) + except FileNotFoundError: + result.error = f"{where} could not be started: {channel.command[0]} does not exist" + return result + except subprocess.TimeoutExpired: + result.error = f"{where} did not finish within {COMMAND_TIMEOUT_SECONDS} s" + return result + except OSError as exc: + result.error = f"{where} could not be started ({exc})" + return result + if ran.returncode != 0: + result.error = f"{where} ended with exit {ran.returncode}. Its stderr:\n{_tail(ran.stderr)}" + return result + for entry in sorted(os.scandir(staging), key=lambda e: e.name): + name = entry.name + path = staging / name + if entry.is_symlink(): + result.skipped.append((name, "symbolic link")) + elif not entry.is_file(follow_symlinks=False): + result.skipped.append((name, "not a regular file")) + elif Path(name).suffix.lower() not in channel.allowed_extensions: + result.skipped.append((name, "extension not allowed")) + else: + try: + upload.sanitize_filename(name) + except ValidationError as exc: + result.skipped.append((name, f"name refused ({exc})")) + continue + _take(root, channel, result, path, name, origin, guard_source=False) + finally: + upload.drop_staging(root, staging) + return result + + +def collect(root: "Path | str", channel: upload.Channel) -> ChannelResult: + """One channel, whatever its handler. Never raises for a failure of the + channel: that is the result's `error`, so the others can still run.""" + try: + if channel.handler == "folder": + return collect_folder(root, channel) + return collect_command(root, channel) + except (ValidationError, OSError) as exc: + failed = ChannelResult(channel.name, channel.handler) + failed.error = str(exc) + return failed diff --git a/tools/chemenu/commands/backup_cmd.py b/tools/chemenu/commands/backup_cmd.py index dfee2db..5b7dba1 100644 --- a/tools/chemenu/commands/backup_cmd.py +++ b/tools/chemenu/commands/backup_cmd.py @@ -112,7 +112,7 @@ def init_command(): "but `.gitkeep`, and the `extra` paths of the `backup` section into " "`//`, then writes the manifest `.chemenu-backup.json` there: time, " "stack version, machine name, and per file its relative path, size and sha256.", - "Never mirrors `mcp-upload/`, `reports/`, `.wikitool-tools.json`, `tools/.venv/`, the " + "Never mirrors `quarantine/`, `reports/`, `.wikitool-tools.json`, `tools/.venv/`, the " "published skills or `.git/` - unless an `extra` path names one of the first ones. The " "secrets file lies outside the instance and is never read.", "Writes and deletes only under `//`. Deletes only a file the previous " diff --git a/tools/chemenu/commands/docs_verify.py b/tools/chemenu/commands/docs_verify.py index 2a54d5a..4237b80 100644 --- a/tools/chemenu/commands/docs_verify.py +++ b/tools/chemenu/commands/docs_verify.py @@ -167,7 +167,7 @@ REQUIRED_IGNORE_CANARIES = ( # The MCP `submit` tool's quarantine (Gitea #32) - stronger than # `incoming/` above: read by no command in the ordinary pipeline, not # only uncommitted. No type subdirectory either, for the same reason. - "mcp-upload/probe.pdf", + "quarantine/probe.pdf", # The live suite's tracker profiles (Gitea #156) - references to credentials, one file per tracker. ".wikitool-tasks.d/probe.json", # `raw capture`'s git cache (Gitea #177) - fetched repositories, recomputable, diff --git a/tools/chemenu/commands/doctor.py b/tools/chemenu/commands/doctor.py index 0015d1a..e370535 100644 --- a/tools/chemenu/commands/doctor.py +++ b/tools/chemenu/commands/doctor.py @@ -641,39 +641,48 @@ def _telemetry_file_deviations() -> list[str]: def check_upload_intake() -> Check: - """Whether the MCP `submit` tool is armed for this checkout, and how full - its quarantine is. + """Which intake channels this checkout has, how many submissions wait in + the quarantine, and whether the quarantine of earlier versions + (`mcp-upload/`) still holds any. Absent is the *safe* default here, unlike `check_publish_remotes`'s "any - push target passes" absence: no `upload` section means the write - path does not exist at all, not that it is unrestricted - so this never - `FAIL`s on a missing section. It does `FAIL` on one that parses to something - invalid, because a broken opt-in must not silently disable the very - limits it exists to enforce. + push target passes" absence: no `intake` section means no channel and no + `submit` tool, not that anything is unrestricted - so this never `FAIL`s on + a missing section. It does `FAIL` on one that parses to something invalid, + because a broken opt-in must not silently disable the very limits it exists + to enforce, and on an `mcp-upload/` with content, which no command reads + any more. """ from chemenu import upload as upload_module + legacy = upload_module.legacy_dir_problem(config.ROOT) + if legacy: + return Check("upload-intake", "FAIL", legacy, "Move the directory as the message says") try: - cfg = upload_module.read_config(config.ROOT) + channels = upload_module.read_channels(config.ROOT) except ChemenuError as exc: return Check( "upload-intake", "FAIL", str(exc), - "Fix the upload section as the message says - a broken one is not treated as " - "'no limits'", + "Fix the intake section as the message says - a broken one is not treated as " + "'no channel'", ) - if cfg is None: + if not channels: return Check( "upload-intake", "OK", - "submit tool not registered - no upload section in the settings", + "no channel - no intake section in the settings, so no submit tool and nothing to collect", ) pending = upload_module.list_submissions(config.ROOT) - return Check( - "upload-intake", "OK", - f"submit tool armed (identity header {cfg.identity_header!r}, up to " - f"{cfg.max_bytes:,} byte(s), {cfg.submissions_per_day}/day and " - f"{cfg.bytes_per_day:,} byte(s)/day per submitter); " - f"{len(pending)} submission(s) waiting in {rel_path(config.UPLOAD_DIR)}", - ) + described = ", ".join(f"{channel.name} ({channel.handler})" for channel in channels.values()) + detail = f"channels: {described}; {len(pending)} submission(s) waiting in {rel_path(config.UPLOAD_DIR)}" + missing = [channel for channel in channels.values() + if channel.handler == "folder" and not (channel.path and channel.path.is_dir())] + if missing: + names = ", ".join(f"{channel.name} ({channel.path})" for channel in missing) + return Check( + "upload-intake", "WARN", f"{detail}; the folder does not exist: {names}", + "Create the folder or correct `path` of the channel in .wikitool-local.json", + ) + return Check("upload-intake", "OK", detail) def check_tasks_provider() -> Check: @@ -1086,9 +1095,11 @@ def run_doctor() -> list[Check]: "Configured, a `WARN` when `/git` is not a bare repository, `origin` does not " "push there, or - with `machine_data` true - no machine-data backup exists yet; " "otherwise `OK` with the age of the last machine-data backup.", - "MCP `submit` tool: whether the `upload` section of the settings is present, absent or " - "malformed, its limits, and how many submissions wait in `mcp-upload/`. Absent is `OK` " - "and means the write path does not exist at all; malformed is a `FAIL`.", + "Intake channels (`upload-intake`): the channels of the `intake` section with their " + "handler, and how many submissions wait in `quarantine/`. Absent is `OK` and means no " + "channel - no `submit` tool, nothing to collect; malformed is a `FAIL`, and so is an " + "`mcp-upload/` of an earlier version that still holds submissions (the message names the " + "command that moves it). A `folder` channel whose `path` does not exist is a `WARN`.", "Task tracker: the `tasks` section present, absent or malformed - absent is `OK` " "(no tracker configured), malformed is a `FAIL`. When `WIKITOOL_TASKS_CONFIG` is set, " "the profile it names is read instead of the `tasks`, `superproductivity` and `caldav` " diff --git a/tools/chemenu/commands/raw/pending.py b/tools/chemenu/commands/raw/pending.py index bd636e5..13bd4af 100644 --- a/tools/chemenu/commands/raw/pending.py +++ b/tools/chemenu/commands/raw/pending.py @@ -131,7 +131,7 @@ def _pending_candidates() -> list[_Candidate]: "A candidate is a top-level entry of `incoming/`: a single `file`, a `bundle` of " "top-level files sharing a stem (a `raw fetch` pair), or " "a `folder` with every file below it. Dotfiles, empty directories and their contents " - "are none; `mcp-upload/` is outside `incoming/` and never listed.", + "are none; `quarantine/` is outside `incoming/` and never listed.", "Order: oldest first by modification time. A bundle or folder counts as new as its " "newest file; a tie goes by name. The mtime is when a document last changed only if " "it was copied with its timestamps kept (`cp -p`, `rsync -a`, an unpacked archive) - " diff --git a/tools/chemenu/commands/upload_cmd.py b/tools/chemenu/commands/upload_cmd.py index 6d340f1..9fd86f1 100644 --- a/tools/chemenu/commands/upload_cmd.py +++ b/tools/chemenu/commands/upload_cmd.py @@ -1,12 +1,14 @@ -"""`wikitool upload list|show|accept|reject` - the human review side of the -MCP submission quarantine (Gitea #32). +"""`wikitool upload list|show|accept|reject|collect` - the quarantine every intake +channel ends in (Gitea #32, #213): the pull from the channels that are not +pushed, and the human review side. Everything that decides *whether* a submission was accepted or written at all lives in `chemenu.upload` - importable from the MCP server, stdlib only. What lives here instead is CLI-only by construction: the **Upload Review Gate** (`accept` refuses with Exit 42 until a human has seen the submission and -re-runs with the printed `--confirm` token), and the two read commands a -reviewer uses to look before clearing it. None of the four is importable +re-runs with the printed `--confirm` token), the two read commands a +reviewer uses to look before clearing it, and `collect`, which reads outside +the quarantine and starts programs. None of the five is importable from `chemenu.mcp.server` - they sit under `chemenu.commands`, the same boundary every other write command is already kept out by (AGENTS.md invariant 6, `instructions/gates.md`). @@ -18,15 +20,16 @@ from typing import Optional import typer -from chemenu import cli_contract, config, upload +from chemenu import cli_contract, collect as collect_module, config, upload from chemenu.commands._util import fail, needs_clearance, rel_path, success from chemenu.errors import ValidationError -app = typer.Typer(help="Review, promote or reject MCP submissions waiting in mcp-upload/.") +app = typer.Typer(help="Collect from the pulled intake channels; review, promote or reject what waits in quarantine/.") def _call(fn, *args, **kwargs): try: + upload.refuse_legacy_dir(config.ROOT) return fn(*args, **kwargs) except ValidationError as exc: fail(str(exc)) @@ -35,7 +38,7 @@ def _call(fn, *args, **kwargs): @app.command("list") @cli_contract.record(cli_contract.CommandRecord( path="upload list", - summary="List every MCP submission currently waiting in the quarantine (`mcp-upload/`).", + summary="List every submission currently waiting in the quarantine (`quarantine/`), whatever channel it came through.", synopsis=(cli_contract.Variant(usage="upload list [--json]"),), properties=cli_contract.Properties( effect=cli_contract.Effect.READ, @@ -44,10 +47,11 @@ def _call(fn, *args, **kwargs): budget=cli_contract.Budget.COUNTED, ), notes=( - "Lists every submission waiting in `mcp-upload/`, oldest id first: id, filename, size, " - "submitter.", - "Only ever non-empty when the `upload` section of `.wikitool-local.json` opts the checkout into the MCP " - "server's `submit` tool.", + "Lists every submission waiting in `quarantine/`, oldest id first: id, filename, size, " + "channel, submitter.", + "Only ever non-empty when the `intake` section of `.wikitool-local.json` declares a channel.", + "Exits 1, before it does anything, while an `mcp-upload/` of an earlier version still holds " + "submissions - the message names the command that moves it.", "A submission directory with a corrupt manifest is silently skipped.", "Never fails; read-only and safe to retry freely.", ), @@ -64,18 +68,18 @@ def _call(fn, *args, **kwargs): def upload_list_command( json_out: bool = typer.Option(False, "--json", help="Print every waiting submission as JSON"), ): - """List every submission currently waiting in mcp-upload/, oldest first.""" + """List every submission currently waiting in quarantine/, oldest first.""" manifests = _call(upload.list_submissions, config.ROOT) if json_out: typer.echo(json_module.dumps(manifests, indent=2)) return if not manifests: - typer.echo("Nothing is waiting in mcp-upload/.") + typer.echo("Nothing is waiting in quarantine/.") return for manifest in manifests: typer.echo( f"{manifest['id']} {manifest['filename']} {manifest['size']}B " - f"from {manifest['submitter']}" + f"via {manifest.get('channel', '?')} from {manifest['submitter']}" ) @@ -92,8 +96,11 @@ def upload_list_command( ), notes=( "Prints one submission's manifest in full: filename, size, sha256, submitter, " - "submitter source (the header name, not a claim the header was honest), submission " - "time.", + "submitter source (the header name or the handler, not a claim the header was honest), " + "submission time, and where it came from: channel, handler, origin (the source file's " + "path, `command:` or `mcp:submit`).", + "A submitter of `unknown` means the channel has no sender to trust - read channel and " + "origin instead.", "What a reviewer reads before `upload accept`.", "Read-only.", ), @@ -119,7 +126,8 @@ def upload_show_command( if json_out: typer.echo(json_module.dumps(manifest, indent=2)) return - for key in ("id", "filename", "size", "sha256", "submitter", "submitter_source", "submitted_at"): + for key in ("id", "filename", "size", "sha256", "submitter", "submitter_source", "submitted_at", + "channel", "handler", "origin"): typer.echo(f"{key}: {manifest.get(key)}") @@ -134,6 +142,9 @@ def _clearance_message(manifest: dict, token: str, stale: Optional[str]) -> str: f" submitter: {manifest['submitter']}", f" submitter_source: {manifest['submitter_source']}", f" submitted_at: {manifest['submitted_at']}", + f" channel: {manifest.get('channel')}", + f" handler: {manifest.get('handler')}", + f" origin: {manifest.get('origin')}", "", ] if stale: @@ -165,8 +176,8 @@ def _clearance_message(manifest: dict, token: str, stale: Optional[str]) -> str: gates=("upload-review",), ), notes=( - "Promotes a submission's file from `mcp-upload//` into `incoming/`, deletes the " - "quarantine directory, and appends an `accepted` event to `mcp-upload/ledger.jsonl`.", + "Promotes a submission's file from `quarantine//` into `incoming/`, deletes the " + "quarantine directory, and appends an `accepted` event to `quarantine/ledger.jsonl`.", "Upload Review Gate: without a matching `--confirm`, exits 42 and prints the manifest " "in full plus the exact re-run line - one submission at a time. The gate check runs " "before anything is moved.", @@ -177,7 +188,7 @@ def _clearance_message(manifest: dict, token: str, stale: Optional[str]) -> str: failures=( cli_contract.Failure( cause="Unknown or malformed submission id, or the submission's file is missing from " - "`mcp-upload//`", + "`quarantine//`", reaction="Fix the named argument and retry once", ), cli_contract.Failure( @@ -261,3 +272,115 @@ def upload_reject_command( """Delete a submission's material, keeping its ledger trail.""" _call(upload.reject, config.ROOT, submission_id, reason) success(f"Rejected '{submission_id}': {reason}") + + +def _result_lines(result: collect_module.ChannelResult) -> list[str]: + head = f"{result.channel} ({result.handler}): " + if result.error: + return [head + "FAILED - " + result.error] + lines = [head + f"{len(result.taken)} taken, {len(result.duplicates)} duplicate, " + f"{len(result.skipped)} skipped, {len(result.failed)} failed"] + lines += [f" taken {name} -> {submission_id}" for name, submission_id in result.taken] + lines += [f" duplicate {name} (already waiting as {submission_id})" for name, submission_id in result.duplicates] + lines += [f" skipped {name} ({reason})" for name, reason in result.skipped] + lines += [f" failed {name} ({reason})" for name, reason in result.failed] + return lines + + +@app.command("collect") +@cli_contract.record(cli_contract.CommandRecord( + path="upload collect", + summary="Pull material from the configured `folder` and `command` intake channels into the " + "quarantine - never past it.", + synopsis=(cli_contract.Variant(usage="upload collect [...]"),), + properties=cli_contract.Properties( + effect=cli_contract.Effect.WRITE, + idempotent=cli_contract.Idempotent.NO, + atomic="No - per file: one copy into `quarantine//` (manifest last), then one removal " + "of the source; a channel's failure does not stop the others", + budget=cli_contract.Budget.COUNTED, + ), + notes=( + "Without arguments, collects from every channel of the `intake` section whose handler is " + "`folder` or `command`; with names, from those channels only. A name no channel has, or " + "the name of the `mcp` channel (which is pushed, not pulled), exits 1 before anything happens.", + "Everything a channel delivers lands in `quarantine//` unchanged, with a manifest that " + "names `channel`, `handler` and `origin`; the submitter is `unknown`. Nothing reaches " + "`incoming/` or `raw/` - that is `upload accept`, behind its gate, and an agent may run " + "`collect` because collecting gives no trust.", + "`folder`: only the top level of `path`. A file is taken once its modification time and, " + "where the platform has it, its creation time are older than `settle_seconds`, its " + "extension and size fit, and its name passes the filename rules. It is removed from the " + "folder only after its copy is fully written and the source still has the size and " + "sha256 that were read; if it changed meanwhile, nothing is kept and it stays. A file whose " + "sha256 already waits in the quarantine is reported as a duplicate and removed from the " + "folder, because its content is kept. Subdirectories, hidden files (names starting with " + "`.` or `~$`, or the hidden or system attribute on Windows), symbolic links, other " + "extensions and oversized files are left untouched and reported.", + "`command`: the program of `commands.` runs without a shell, with stdin closed, a " + "300 s timeout and an empty staging directory under `quarantine/.staging/` as its last " + "argument. On exit 0 each regular file at the top of that directory with an allowed " + "extension and size becomes a submission; on anything else nothing is taken and the end of " + "stderr is printed. The staging directory is always removed.", + "Channels are independent: when one fails the others still run, and the exit status is 1 " + "if any channel or file failed. With no pulled channel it prints `nothing to collect` " + "and exits 0.", + "Exits 1, before it does anything, while an `mcp-upload/` of an earlier version still holds " + "submissions, and when the `intake` section is malformed.", + "Not idempotent: a second run finds the folder emptied.", + ), + failures=( + cli_contract.Failure( + cause="Unknown channel name, an `mcp` channel as argument, a malformed `intake` section, " + "or an `mcp-upload/` that still holds submissions", + reaction="Fix the named cause and retry once", + ), + cli_contract.Failure( + cause="A channel or a file failed (folder missing, program exited non-zero, timed out or " + "could not be started, a source that could not be removed)", + reaction="Not fixed by retrying unchanged - read the per-channel lines, fix the channel's " + "folder or program, then run `collect ` for that channel only", + ), + ), + examples=( + "tools/wikitool upload collect", + "tools/wikitool upload collect onedrive", + ), + never=( + "Never follow `collect` with `upload accept` and a `--confirm` token the user has not seen " + "and approved.", + ), + see_also=( + "`wikitool upload list` - what waits for review afterwards", + "`wikitool upload accept` - the gate out of the quarantine", + "`docs/how-to/collect-from-a-synced-folder.md`", + ), +)) +def upload_collect_command( + names: Optional[list[str]] = typer.Argument( + None, help="Channels to collect from; all pulled channels when none is named" + ), +): + """Pull from the folder and command channels into the quarantine.""" + channels = _call(upload.read_channels, config.ROOT) + pulled = {name: channel for name, channel in channels.items() if channel.pulled} + chosen = list(names or []) + for name in chosen: + if name not in channels: + fail(f"No channel '{name}' in the `intake` section (channels: " + f"{', '.join(channels) or 'none'}). Nothing was collected.") + if not channels[name].pulled: + fail(f"The channel '{name}' has the handler mcp: it is pushed through the `submit` tool, " + "there is nothing to collect. Nothing was collected.") + selected = [pulled[name] for name in dict.fromkeys(chosen)] if chosen else list(pulled.values()) + if not selected: + typer.echo("nothing to collect") + return + ok = True + for channel in selected: + result = collect_module.collect(config.ROOT, channel) + ok = ok and result.ok + for line in _result_lines(result): + typer.echo(line) + if not ok: + raise typer.Exit(code=1) diff --git a/tools/chemenu/config.py b/tools/chemenu/config.py index bd13198..636dcb9 100644 --- a/tools/chemenu/config.py +++ b/tools/chemenu/config.py @@ -62,6 +62,9 @@ def _root() -> Path: return Path(assigned) if assigned is not None else resolve_root() +# The quarantine's name; `UPLOAD_DIR` below is that directory under the root. +UPLOAD_DIRNAME = "quarantine" + # Everything under the root, as a name -> relative-path table rather than as # assignments. One place to read, and the only place that has to know a derived # path exists at all. @@ -72,10 +75,10 @@ _DERIVED = { "REPORTS_DIR": ("reports",), "WORK_DIR": ("work",), "INSTRUCTIONS_DIR": ("instructions",), - # The MCP upload quarantine (Gitea #32) - never `raw/` and never `incoming/`, - # see raw/CONTRACT.md "Getting a file in". Gitignored; the server process is - # the only writer. - "UPLOAD_DIR": ("mcp-upload",), + # The quarantine every intake channel ends in (Gitea #32, #213) - never `raw/` + # and never `incoming/`, see raw/CONTRACT.md "Getting a file in". Gitignored + # twice over: the root `.gitignore` and a `.gitignore` of its own. + "UPLOAD_DIR": (UPLOAD_DIRNAME,), # Generated copies of the skill directories under `instructions/`. Both are # gitignored: they are build output, and a fresh clone publishes them with # `wikitool instructions sync` (see instructions/bootstrap.md). @@ -83,6 +86,10 @@ _DERIVED = { "CLAUDE_SKILLS_DIR": (".claude", "skills"), } +# What the quarantine was called before 10.0.0. Nothing reads or writes it any +# more; a directory of this name with content is reported, never moved. +LEGACY_UPLOAD_DIRNAME = "mcp-upload" + # The generated files, derived from `KB_DIR` rather than from the root: a # caller that repoints only the corpus directory must not be left with a log # and a catalog belonging to a different tree. diff --git a/tools/chemenu/config_shapes.py b/tools/chemenu/config_shapes.py index 4f3d8c5..5cd6861 100644 --- a/tools/chemenu/config_shapes.py +++ b/tools/chemenu/config_shapes.py @@ -136,6 +136,11 @@ class Shape: examples: tuple[dict, ...] absent: str = "" malformed: str = "" + # Judges the section as a whole once every field passed - for a rule that + # spans fields, such as a reference from one field to an entry of another. + # Gets the coerced values; returns a message that starts with the dotted + # place it is about (the section key is put in front), or None. + section_check: Optional[Callable[[dict], Optional[str]]] = None def field(self, name: str) -> Field: for candidate in self.fields: @@ -278,6 +283,119 @@ def fetchers_problem(value: dict) -> Optional[str]: return None +INTAKE_HANDLER_MCP = "mcp" +INTAKE_HANDLER_FOLDER = "folder" +INTAKE_HANDLER_COMMAND = "command" +INTAKE_HANDLERS = (INTAKE_HANDLER_MCP, INTAKE_HANDLER_FOLDER, INTAKE_HANDLER_COMMAND) +INTAKE_SETTLE_SECONDS = 60 +INTAKE_DEFAULT_IDENTITY_HEADER = "X-Forwarded-User" +# The keys every channel has, then the ones only its handler has. +_CHANNEL_COMMON = ("handler", "max_bytes", "allowed_extensions") +_CHANNEL_KEYS = { + INTAKE_HANDLER_MCP: ("identity_header", "quota_submissions_per_day", "quota_bytes_per_day"), + INTAKE_HANDLER_FOLDER: ("path", "settle_seconds"), + INTAKE_HANDLER_COMMAND: ("command",), +} +_CHANNEL_REQUIRED = { + INTAKE_HANDLER_MCP: ("quota_submissions_per_day", "quota_bytes_per_day"), + INTAKE_HANDLER_FOLDER: ("path",), + INTAKE_HANDLER_COMMAND: ("command",), +} + + +def mcp_channel_keys() -> tuple[str, ...]: + """The keys a channel with the handler `mcp` takes.""" + return _CHANNEL_COMMON[1:] + _CHANNEL_KEYS[INTAKE_HANDLER_MCP] + + +def _is_int(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _channel_problem(name: str, channel: Any) -> Optional[str]: + where = f"the channel {name!r}" + if not isinstance(channel, dict): + return f"{where} must be an object" + handler = channel.get("handler") + if handler not in INTAKE_HANDLERS: + return f"{where} needs `handler` to be one of {', '.join(INTAKE_HANDLERS)}; got {handler!r}" + own = _CHANNEL_KEYS[handler] + stray = sorted(set(channel) - set(_CHANNEL_COMMON) - set(own)) + if stray: + return (f"{where} (handler {handler}) has {', '.join(f'`{key}`' for key in stray)}, which " + f"that handler does not take - its fields are {', '.join(_CHANNEL_COMMON + own)}") + for key in _CHANNEL_REQUIRED[handler] + ("max_bytes", "allowed_extensions"): + if key not in channel: + return f"{where} (handler {handler}) is missing `{key}`" + if not _is_int(channel["max_bytes"]) or channel["max_bytes"] <= 0: + return f"{where}: `max_bytes` must be a positive integer" + exts = channel["allowed_extensions"] + if (not isinstance(exts, list) or not exts + or not all(isinstance(ext, str) and ext.startswith(".") and len(ext) > 1 for ext in exts)): + return f"{where}: `allowed_extensions` must be a non-empty array of strings that each start with `.`" + if handler == INTAKE_HANDLER_MCP: + header = channel.get("identity_header", INTAKE_DEFAULT_IDENTITY_HEADER) + if not isinstance(header, str) or not header.strip(): + return f"{where}: `identity_header` must be a non-empty string" + for key in ("quota_submissions_per_day", "quota_bytes_per_day"): + if not _is_int(channel[key]) or channel[key] <= 0: + return f"{where}: `{key}` must be a positive integer" + elif handler == INTAKE_HANDLER_FOLDER: + path = channel["path"] + if not isinstance(path, str) or not path.strip(): + return f"{where}: `path` must be a non-empty string" + problem = absolute_path_problem(path) + if problem: + return f"{where}: `path` {problem}" + settle = channel.get("settle_seconds", INTAKE_SETTLE_SECONDS) + if not _is_int(settle) or settle < 0: + return f"{where}: `settle_seconds` must be an integer of 0 or more" + else: + ref = channel["command"] + if not isinstance(ref, str) or not _FETCHER_NAME.fullmatch(ref): + return (f"{where}: `command` names an entry of `commands` - lower-case letters, digits and " + "hyphens") + return None + + +def intake_channels_problem(value: dict) -> Optional[str]: + if not value: + return "must hold at least one channel" + mcp = [] + for name, channel in value.items(): + if not _FETCHER_NAME.fullmatch(name): + return (f"has the channel name {name!r} - a channel's name is lower-case letters, digits " + "and hyphens") + problem = _channel_problem(name, channel) + if problem: + return problem + if channel["handler"] == INTAKE_HANDLER_MCP: + mcp.append(name) + if len(mcp) > 1: + return (f"has {len(mcp)} channels with handler mcp ({', '.join(mcp)}) - the MCP server has " + "one `submit` tool, so at most one channel takes that handler") + return None + + +def intake_commands_problem(value: dict) -> Optional[str]: + for name, command in value.items(): + if not _FETCHER_NAME.fullmatch(name): + return f"has the name {name!r} - a command's name is lower-case letters, digits and hyphens" + if (not isinstance(command, list) or not command + or not all(isinstance(part, str) and part for part in command)): + return f"{name} must be a non-empty array of non-empty strings: the program, then its arguments" + return None + + +def intake_section_problem(values: dict) -> Optional[str]: + commands = values.get("commands") or {} + for name, channel in values["channels"].items(): + if channel["handler"] == INTAKE_HANDLER_COMMAND and channel["command"] not in commands: + return (f"channels.{name}.command names {channel['command']!r}, which `commands` does not " + "define") + return None + + def _positive(value: int) -> Optional[str]: return None if value > 0 else "must be positive" @@ -359,39 +477,66 @@ REMOTES = Shape( "read as \"no restriction\".", ) -UPLOAD = Shape( - key="upload", - title="MCP submissions", - purpose="Turns on the MCP read server's `submit` tool and sets its limits.", +INTAKE = Shape( + key="intake", + title="Intake channels", + purpose="Declares the channels material reaches the quarantine through: the MCP server's " + "`submit` tool, a folder a sync client fills, or a program of this checkout. Whatever a " + "channel delivers waits in `quarantine/` until a human accepts it.", fields=( Field( - "identity_header", "string", "The request header the authenticating middleware sets to " - "the submitter's identity on every request.", - default="X-Forwarded-User", json_type="string", coerce=str, empty_is_missing=True, - lenient="An empty value means the default.", tier=CHECKOUT, gate=True, - ), - positive_integer("max_bytes", "Largest single submission, in bytes.", required=True, - tier=CHECKOUT, gate=True), - Field( - "allowed_extensions", "array of strings", "The file extensions `submit` accepts, each " - "starting with `.`. Compared case-insensitively.", - required=True, json_type="array", coerce=extensions, check=dotted_extensions, - lenient="Entries are read as strings and lower-cased; duplicates collapse.", + "channels", "object", "Channel name (lower-case letters, digits and hyphens) to " + "channel. The name goes into the manifest of what the channel delivers and is the " + "argument of `upload collect`. Every channel has `handler` (`mcp`, `folder` or " + "`command`), `max_bytes` (the largest single file) and `allowed_extensions` (each " + "starting with `.`, compared case-insensitively). Then per handler: `mcp` takes " + "`identity_header` (the request header the authenticating middleware sets to the " + f"submitter's identity, default `{INTAKE_DEFAULT_IDENTITY_HEADER}`) and, required, " + "`quota_submissions_per_day` and `quota_bytes_per_day` (per submitter, over any " + "rolling 24 hours); `folder` takes `path` (required: an absolute path outside the " + "instance, read without descending into subdirectories) and `settle_seconds` (default " + f"{INTAKE_SETTLE_SECONDS}: a file whose modification or creation time is younger is " + "left alone); `command` takes `command` (required: the name of an entry of " + "`commands`). At most one channel has the handler `mcp`. A field the handler does not " + "take is an error.", + required=True, json_type="object", coerce=object_value, check=intake_channels_problem, tier=CHECKOUT, gate=True, ), - positive_integer("quota_submissions_per_day", "Submissions one submitter may make over any " - "rolling 24 hours.", required=True, tier=CHECKOUT, gate=True), - positive_integer("quota_bytes_per_day", "Bytes one submitter may submit over any rolling " - "24 hours.", required=True, tier=CHECKOUT, gate=True), + Field( + "commands", "object", "Command name to command: an array of the program and its " + "arguments, run by the `command` handler without a shell, with stdin closed and a " + "timeout of 300 s, and a staging directory appended as the last argument. Exit 0 " + "turns each regular file at the top level of that directory into a submission; " + "anything else takes nothing over and prints the end of the program's stderr.", + default={}, json_type="object", coerce=object_value, check=intake_commands_problem, + tier=CHECKOUT, names_command=True, + ), ), examples=({ - "identity_header": "X-Forwarded-User", "max_bytes": 10485760, - "allowed_extensions": [".md", ".pdf"], - "quota_submissions_per_day": 20, "quota_bytes_per_day": 52428800, + "channels": { + "mcp": { + "handler": "mcp", "identity_header": INTAKE_DEFAULT_IDENTITY_HEADER, + "max_bytes": 10485760, "allowed_extensions": [".md", ".pdf"], + "quota_submissions_per_day": 20, "quota_bytes_per_day": 52428800, + }, + "onedrive": { + "handler": "folder", + "path": "C:\\Users\\\\OneDrive - \\Chemenu-Eingang", + "max_bytes": 52428800, "allowed_extensions": [".pdf", ".docx", ".msg", ".eml", ".md"], + "settle_seconds": 60, + }, + "scanner": { + "handler": "command", "command": "scanner-pull", + "max_bytes": 52428800, "allowed_extensions": [".pdf"], + }, + }, + "commands": {"scanner-pull": ["pwsh", "-NoProfile", "-File", "C:\\tools\\scanner-pull.ps1"]}, },), - absent="The `submit` tool is not registered: the write path does not exist.", - malformed="An error. The MCP server does not start and `doctor` reports `FAIL`. A broken section " - "is never read as \"no limits\".", + absent="No channel: the `submit` tool is not registered and `upload collect` has nothing to " + "collect. A valid state; `doctor` reports `OK`.", + malformed="An error. The MCP server does not start, `upload collect` stops with exit 1 and " + "`doctor` reports `FAIL`. A broken section is never read as \"no channel\".", + section_check=intake_section_problem, ) TASKS = Shape( @@ -672,7 +817,7 @@ BACKUP = Shape( # `.wikitool-local.json` (`chemenu.settings`), read at call time so a test can # stand in a section of its own. SETTINGS_SECTIONS: tuple[Shape, ...] = ( - TELEMETRY, REMOTES, UPLOAD, TASKS, SUPERPRODUCTIVITY, CALDAV, SITES, FETCH, BACKUP, + TELEMETRY, REMOTES, INTAKE, TASKS, SUPERPRODUCTIVITY, CALDAV, SITES, FETCH, BACKUP, ) # The sections `WIKITOOL_TASKS_CONFIG` replaces as a whole: a profile names a @@ -747,6 +892,10 @@ def check(shape: Shape, data: Any) -> Checked: )) out.values[spec.name] = value out.given.add(spec.name) + if shape.section_check is not None and not out.problems: + problem = shape.section_check(out.values) + if problem: + out.problems.append(Problem("", problem)) out.unknown = sorted(set(data) - {spec.name for spec in shape.fields}) return out diff --git a/tools/chemenu/legacy_settings.py b/tools/chemenu/legacy_settings.py index e23632d..66ec247 100644 --- a/tools/chemenu/legacy_settings.py +++ b/tools/chemenu/legacy_settings.py @@ -10,7 +10,7 @@ into `wikitool.json`, `.wikitool-local.json` and the secrets file - |---|---| | `.wikitool-telemetry.json` | section `telemetry` | | `.wikitool-remotes.json` | section `remotes` | -| `.wikitool-upload.json` | section `upload` (`quota.*` is flat: `quota_*`) | +| `.wikitool-upload.json` | section `intake`, channel `mcp` with `handler: "mcp"` (`quota.*` is flat: `quota_*`) | | `.wikitool-tasks.json` | sections `tasks` (`thresholds.*` is flat), `superproductivity`, `caldav` | What the migration promises, and how it keeps it: @@ -49,7 +49,7 @@ TASKS_FILE = ".wikitool-tasks.json" FILES = { "telemetry": TELEMETRY_FILE, "remotes": REMOTES_FILE, - "upload": UPLOAD_FILE, + "intake": UPLOAD_FILE, "tasks": TASKS_FILE, } @@ -199,7 +199,14 @@ def read_plan(root: "Path | str") -> Plan: if path.exists(): plan.files.append(path) data = _flat(_read_old(path, lenient=False, plan=plan), "quota", "quota_", UPLOAD_FILE) - _take(plan, config_shapes.UPLOAD, data, file=UPLOAD_FILE) + if isinstance(data, dict): + known = set(config_shapes.mcp_channel_keys()) + ignored = [key for key in data if key not in known and key != "schema"] + if ignored: + plan.notes.append(f"{UPLOAD_FILE}: {', '.join(ignored)} was not read before and is not migrated") + data = {key: value for key, value in data.items() if key in known} + data = {"handler": config_shapes.INTAKE_HANDLER_MCP, **data} + _take(plan, config_shapes.INTAKE, {"channels": {"mcp": data}}, file=UPLOAD_FILE) path = root / TASKS_FILE if path.exists(): diff --git a/tools/chemenu/mcp/server.py b/tools/chemenu/mcp/server.py index 9524bdc..4e270c7 100644 --- a/tools/chemenu/mcp/server.py +++ b/tools/chemenu/mcp/server.py @@ -1,5 +1,5 @@ """The MCP server: `search`, `types`, `lint`, `status` over `kb/`, plus an -opt-in write path, `submit`, into a quarantine (Gitea #32). +opt-in write path, `submit`, into a quarantine (Gitea #32, #213). Chemenu's second consumer. The CLI and this are two adapters over one core - `chemenu.api.Corpus` - so a question answered here and the same question asked @@ -15,12 +15,12 @@ test asserts it by importing this module in a clean interpreter and looking at `sys.modules`. **The sixth, `submit`, writes by a positive list instead of an absence.** It -exists only when the `upload` section of `.wikitool-local.json` opts this -checkout into it (absent by default) - not merely hidden, but +exists only when the `intake` section of `.wikitool-local.json` declares a +channel with the handler `mcp` (absent by default) - not merely hidden, but never registered on the server, the same distinction #19 draws for every other write function. When it is registered, every byte it writes still goes through exactly one choke point, `chemenu.upload._write_atomic_within`, which -resolves the target and refuses anything outside `mcp-upload/`. That +resolves the target and refuses anything outside `quarantine/`. That directory is read by no other command in the ordinary pipeline - promoting a submission out of it is `wikitool upload accept`, gated (Exit 42) and run by a human, never by this process. See `instructions/ingest-queue.md`. @@ -128,18 +128,21 @@ def build_server( if check_trace: check_trace_destination(corpus.root) - # A malformed `upload` section is a start failure, not "no limits": + # A malformed `intake` section is a start failure, not "no limits": # this section decides whether an unauthenticated write path is offered at # all, so it is read once, up front, rather than lazily inside the tool # closure where a broken section would only surface on the first `submit` # call. `upload_cfg` is None exactly when the tool below is not defined. + # So is a quarantine of an earlier version that still holds submissions: + # starting beside it would leave them where no command looks. + upload.refuse_legacy_dir(corpus.root) upload_cfg = upload.read_config(corpus.root) read_only_note = ( "Five tools are structurally read-only - there is no tool that writes among them." if upload_cfg is None else "Five tools are structurally read-only. The sixth, 'submit', writes only into " - "a quarantine ('mcp-upload/') no other tool or command reads - nothing here can " + "a quarantine ('quarantine/') no other tool or command reads - nothing here can " "reach kb/. A submission needs a human to promote it (see instructions/ingest-queue.md)." ) server = MCPServer( diff --git a/tools/chemenu/reference_doc.py b/tools/chemenu/reference_doc.py index 9e54055..3c65512 100644 --- a/tools/chemenu/reference_doc.py +++ b/tools/chemenu/reference_doc.py @@ -67,7 +67,7 @@ CONFIG_REGIONS = ( ("reference-config-tasks", "TASKS"), ("reference-config-superproductivity", "SUPERPRODUCTIVITY"), ("reference-config-caldav", "CALDAV"), - ("reference-config-upload", "UPLOAD"), + ("reference-config-intake", "INTAKE"), ("reference-config-sites", "SITES"), ("reference-config-fetch", "FETCH"), ("reference-config-backup", "BACKUP"), diff --git a/tools/chemenu/tests/test_backup.py b/tools/chemenu/tests/test_backup.py index f0c8559..924d04b 100644 --- a/tools/chemenu/tests/test_backup.py +++ b/tools/chemenu/tests/test_backup.py @@ -47,7 +47,7 @@ def instance(tmp_path, monkeypatch): (root / "incoming").mkdir() (root / "incoming" / ".gitkeep").write_text("", encoding="utf-8") (root / ".gitignore").write_text( - "/incoming/*\n!/incoming/.gitkeep\n/.wikitool-local.json\n/ENVIRONMENT.md\n/mcp-upload/\n" + "/incoming/*\n!/incoming/.gitkeep\n/.wikitool-local.json\n/ENVIRONMENT.md\n/quarantine/\n" "/reports/\n/.wikitool-tools.json\n", encoding="utf-8") (root / "README.md").write_text("init\n", encoding="utf-8") _git(root, "add", "-A") @@ -222,7 +222,7 @@ def _fill(root: Path) -> None: (root / "incoming" / "paper.pdf").write_bytes(b"%PDF-1.7 fixture") (root / "incoming" / "old").mkdir() (root / "incoming" / "old" / "note.md").write_text("note\n", encoding="utf-8") - for relative in ("mcp-upload/sub-1/x.md", "reports/lint.md", ".wikitool-tools.json"): + for relative in ("quarantine/sub-1/x.md", "reports/lint.md", ".wikitool-tools.json"): (root / relative).parent.mkdir(parents=True, exist_ok=True) (root / relative).write_text("not mirrored\n", encoding="utf-8") (root / ".claude").mkdir() diff --git a/tools/chemenu/tests/test_collect.py b/tools/chemenu/tests/test_collect.py new file mode 100644 index 0000000..b3506f7 --- /dev/null +++ b/tools/chemenu/tests/test_collect.py @@ -0,0 +1,297 @@ +"""Tests for the pulled intake channels (`chemenu/collect.py`) and `wikitool +upload collect`: the `folder` and `command` handlers, and what the command does +across several channels.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest +import typer + +from chemenu import collect, config, upload +from chemenu.commands.upload_cmd import upload_collect_command, upload_list_command + +OLD = time.time() - 3600 + + +@pytest.fixture +def root(tmp_path, monkeypatch): + instance = tmp_path / "instance" + instance.mkdir() + (instance / "incoming").mkdir() + monkeypatch.setattr(config, "ROOT", instance) + return instance + + +@pytest.fixture +def drop(tmp_path): + folder = tmp_path / "drop" + folder.mkdir() + return folder + + +def _put(folder: Path, name: str, data: bytes = b"content\n", age: float = 3600) -> Path: + path = folder / name + path.write_bytes(data) + stamp = time.time() - age + os.utime(path, (stamp, stamp)) + return path + + +def _folder_channel(drop: Path, **overrides) -> upload.Channel: + fields = dict( + name="onedrive", handler="folder", max_bytes=1024, allowed_extensions=(".md", ".pdf"), + path=drop, settle_seconds=60, + ) + fields.update(overrides) + return upload.Channel(**fields) + + +def _command_channel(argv: list[str], **overrides) -> upload.Channel: + fields = dict( + name="mail", handler="command", max_bytes=1024, allowed_extensions=(".md", ".pdf"), + command_name="scan", command=tuple(argv), + ) + fields.update(overrides) + return upload.Channel(**fields) + + +def _script(body: str) -> list[str]: + return [sys.executable, "-c", f"import sys, pathlib\nout = pathlib.Path(sys.argv[1])\n{body}"] + + +def _write_intake(root: Path, body: dict) -> None: + (root / config.LOCAL_SETTINGS_FILENAME).write_text( + json.dumps({"schema": 1, "intake": body}), encoding="utf-8") + + +def _tree(folder: Path) -> dict[str, bytes]: + return {str(p.relative_to(folder)): p.read_bytes() for p in sorted(folder.rglob("*")) if p.is_file()} + + +def _manifests(root: Path) -> list[dict]: + return upload.list_submissions(root) + + +# --- folder ----------------------------------------------------------------- + +def test_a_folder_pull_stores_each_file_and_empties_the_source(root, drop): + _put(drop, "a.md", b"alpha\n") + _put(drop, "b.pdf", b"bravo\n") + result = collect.collect(root, _folder_channel(drop)) + assert result.ok and len(result.taken) == 2 + assert _tree(drop) == {} + manifests = {m["filename"]: m for m in _manifests(root)} + assert set(manifests) == {"a.md", "b.pdf"} + first = manifests["a.md"] + assert (first["channel"], first["handler"]) == ("onedrive", "folder") + assert first["origin"] == str(drop / "a.md") + assert (first["submitter"], first["submitter_source"]) == ("unknown", "folder") + assert (root / "quarantine" / first["id"] / "a.md").read_bytes() == b"alpha\n" + + +def test_a_folder_pull_leaves_what_it_does_not_take_untouched(root, drop): + (drop / "sub").mkdir() + (drop / "sub" / "inner.md").write_bytes(b"inner") + _put(drop, ".x.pdf", b"hidden") + _put(drop, "~$a.docx", b"lock") + _put(drop, "notes.exe", b"bad extension") + _put(drop, "big.md", b"x" * 2000) + target = _put(drop, "real.md", b"target") + link = drop / "link.md" + link.symlink_to(target) + target.unlink() + before = _tree(drop) + result = collect.collect(root, _folder_channel(drop)) + assert result.ok and result.taken == [] and _manifests(root) == [] + assert _tree(drop) == before + assert link.is_symlink() + reasons = dict(result.skipped) + assert reasons["sub"] == "subdirectory" + assert reasons[".x.pdf"] == "hidden" and reasons["~$a.docx"] == "hidden" + assert reasons["notes.exe"] == "extension not allowed" + assert reasons["link.md"] == "symbolic link" + assert "max_bytes" in reasons["big.md"] + + +def test_a_file_younger_than_settle_seconds_waits_for_the_next_run(root, drop): + _put(drop, "fresh.md", b"arriving", age=5) + result = collect.collect(root, _folder_channel(drop, settle_seconds=60)) + assert result.taken == [] and "still in motion" in dict(result.skipped)["fresh.md"] + assert (drop / "fresh.md").read_bytes() == b"arriving" + os.utime(drop / "fresh.md", (OLD, OLD)) + later = collect.collect(root, _folder_channel(drop, settle_seconds=60)) + assert [name for name, _ in later.taken] == ["fresh.md"] + + +def test_a_source_that_changes_during_the_copy_is_kept_and_nothing_is_stored(root, drop, monkeypatch): + source = _put(drop, "a.md", b"one\n") + monkeypatch.setattr(collect, "source_matches", lambda *args: False) + result = collect.collect(root, _folder_channel(drop)) + assert result.taken == [] and "changed while it was copied" in dict(result.skipped)["a.md"] + assert source.read_bytes() == b"one\n" + assert _manifests(root) == [] + assert [p for p in (root / "quarantine").iterdir() if p.is_dir()] == [] + + +def test_a_duplicate_is_reported_with_the_existing_id_and_the_source_is_removed(root, drop): + _put(drop, "a.md", b"same\n") + first = collect.collect(root, _folder_channel(drop)) + existing = first.taken[0][1] + _put(drop, "again.md", b"same\n") + second = collect.collect(root, _folder_channel(drop)) + assert second.taken == [] and second.duplicates == [("again.md", existing)] + assert not (drop / "again.md").exists() + assert len(_manifests(root)) == 1 + + +def test_a_missing_folder_fails_the_channel(root, tmp_path): + result = collect.collect(root, _folder_channel(tmp_path / "gone")) + assert not result.ok and "does not exist" in result.error + + +# --- command ---------------------------------------------------------------- + +def test_a_command_hands_over_the_files_it_left_in_the_staging_directory(root): + channel = _command_channel(_script("(out / 'mail.md').write_bytes(b'from a script\\n')")) + result = collect.collect(root, channel) + assert result.ok and [name for name, _ in result.taken] == ["mail.md"] + manifest = _manifests(root)[0] + assert (manifest["channel"], manifest["handler"], manifest["origin"]) == ("mail", "command", "command:scan") + assert (manifest["submitter"], manifest["submitter_source"]) == ("unknown", "command") + assert not (root / "quarantine" / ".staging").exists() + + +def test_a_failing_command_takes_nothing_and_shows_its_stderr_tail(root): + channel = _command_channel(_script( + "(out / 'half.md').write_bytes(b'x')\nsys.stderr.write('boom ' * 600 + 'THE END')\nsys.exit(3)")) + result = collect.collect(root, channel) + assert not result.ok and result.taken == [] + assert "exit 3" in result.error and result.error.rstrip().endswith("THE END") + assert len(result.error) < 2500 + assert _manifests(root) == [] + assert not (root / "quarantine" / ".staging").exists() + + +def test_a_command_that_times_out_fails_its_channel_and_cleans_up(root, monkeypatch): + monkeypatch.setattr(collect, "COMMAND_TIMEOUT_SECONDS", 1) + channel = _command_channel(_script("(out / 'a.md').write_bytes(b'x')\nimport time\ntime.sleep(30)")) + result = collect.collect(root, channel) + assert not result.ok and "did not finish" in result.error + assert _manifests(root) == [] + assert not (root / "quarantine" / ".staging").exists() + + +def test_a_command_that_does_not_exist_fails_its_channel(root, tmp_path): + result = collect.collect(root, _command_channel([str(tmp_path / "nope.sh")])) + assert not result.ok and "does not exist" in result.error + + +def test_a_command_result_skips_what_is_not_a_plain_allowed_file(root): + channel = _command_channel(_script( + "(out / 'sub').mkdir()\n(out / 'x.exe').write_bytes(b'x')\n(out / 'ok.md').write_bytes(b'ok')")) + result = collect.collect(root, channel) + assert [name for name, _ in result.taken] == ["ok.md"] + assert dict(result.skipped) == {"sub": "not a regular file", "x.exe": "extension not allowed"} + + +# --- upload collect --------------------------------------------------------- + +def _two_channels(root, drop): + _write_intake(root, { + "commands": {"scan": _script("(out / 'mail.md').write_bytes(b'mail\\n')")}, + "channels": { + "onedrive": {"handler": "folder", "path": str(drop), "max_bytes": 1024, + "allowed_extensions": [".md"]}, + "mail": {"handler": "command", "command": "scan", "max_bytes": 1024, + "allowed_extensions": [".md"]}, + "server": {"handler": "mcp", "max_bytes": 1024, "allowed_extensions": [".md"], + "quota_submissions_per_day": 1, "quota_bytes_per_day": 1000}, + }, + }) + + +def test_collect_without_a_pull_channel_has_nothing_to_do(root, capsys): + upload_collect_command(names=None) + assert "nothing to collect" in capsys.readouterr().out + + +def test_collect_pulls_every_pull_channel_and_reports_per_channel(root, drop, capsys): + _two_channels(root, drop) + _put(drop, "a.md", b"alpha\n") + upload_collect_command(names=None) + out = capsys.readouterr().out + assert "onedrive (folder)" in out and "mail (command)" in out + assert {m["channel"] for m in _manifests(root)} == {"onedrive", "mail"} + + +def test_collect_names_one_channel_and_leaves_the_others(root, drop): + _two_channels(root, drop) + _put(drop, "a.md", b"alpha\n") + upload_collect_command(names=["mail"]) + assert {m["channel"] for m in _manifests(root)} == {"mail"} + assert (drop / "a.md").exists() + + +def test_a_failing_channel_does_not_stop_the_others_but_the_exit_is_1(root, drop, tmp_path, capsys): + _two_channels(root, drop) + body = json.loads((root / config.LOCAL_SETTINGS_FILENAME).read_text()) + body["intake"]["channels"]["onedrive"]["path"] = str(tmp_path / "gone") + (root / config.LOCAL_SETTINGS_FILENAME).write_text(json.dumps(body), encoding="utf-8") + with pytest.raises(typer.Exit) as excinfo: + upload_collect_command(names=None) + assert excinfo.value.exit_code == 1 + assert "FAILED" in capsys.readouterr().out + assert {m["channel"] for m in _manifests(root)} == {"mail"} + + +@pytest.mark.parametrize("names", [["nope"], ["server"], ["mail", "nope"], ["mail", "server"]]) +def test_an_unknown_or_mcp_argument_exits_1_before_any_effect(root, drop, names): + _two_channels(root, drop) + _put(drop, "a.md", b"alpha\n") + before = _tree(drop) + with pytest.raises(typer.Exit) as excinfo: + upload_collect_command(names=names) + assert excinfo.value.exit_code == 1 + assert _tree(drop) == before + assert _manifests(root) == [] + + +def test_a_malformed_intake_section_exits_1(root): + _write_intake(root, {"channels": {"a": {"handler": "folder"}}}) + with pytest.raises(typer.Exit) as excinfo: + upload_collect_command(names=None) + assert excinfo.value.exit_code == 1 + + +def test_the_command_refuses_while_a_legacy_quarantine_holds_submissions(root, drop): + _two_channels(root, drop) + _put(drop, "a.md", b"alpha\n") + (root / "mcp-upload" / "sub-1").mkdir(parents=True) + (root / "mcp-upload" / "sub-1" / "a.md").write_bytes(b"waiting") + with pytest.raises(typer.Exit) as excinfo: + upload_collect_command(names=None) + assert excinfo.value.exit_code == 1 + assert (drop / "a.md").exists() + with pytest.raises(typer.Exit): + upload_list_command(json_out=False) + + +# --- the quarantine's own .gitignore ---------------------------------------- + +def _git(root: Path, *args: str) -> str: + return subprocess.run(["git", *args], cwd=root, check=True, capture_output=True, text=True).stdout + + +def test_collected_material_shows_in_no_git_status_even_without_a_root_entry(root, drop): + _git(root, "init", "-q") + _put(drop, "a.md", b"alpha\n") + collect.collect(root, _folder_channel(drop)) + collect.collect(root, _command_channel(_script("(out / 'm.md').write_bytes(b'mail')"))) + assert (root / "quarantine" / ".gitignore").read_text().strip() == "*" + assert _git(root, "status", "--porcelain", "--untracked-files=all") == "" diff --git a/tools/chemenu/tests/test_config_shapes.py b/tools/chemenu/tests/test_config_shapes.py index 7ff0d63..44b4e12 100644 --- a/tools/chemenu/tests/test_config_shapes.py +++ b/tools/chemenu/tests/test_config_shapes.py @@ -29,10 +29,10 @@ runner = CliRunner() PACKAGE = Path(config_shapes.__file__).parent -UPLOAD_OK = { - "identity_header": "X-User", "max_bytes": 1000, "allowed_extensions": [".md", ".PDF"], +INTAKE_OK = {"channels": {"mcp": { + "handler": "mcp", "identity_header": "X-User", "max_bytes": 1000, "allowed_extensions": [".md", ".PDF"], "quota_submissions_per_day": 5, "quota_bytes_per_day": 5000, -} +}}} TASKS_OK = {"provider": "caldav", "stalled_waiting_days": 14, "unpaged_project_weeks": 3, "someday_stale_months": 5} CALDAV_OK = {"url": "https://dav.example/", "username": "u", "app_password": "pw", @@ -78,7 +78,7 @@ def test_the_telemetry_policy_reads_the_record(): def test_one_record_per_section(): assert [shape.key for shape in config_shapes.SETTINGS_SECTIONS] == [ - "telemetry", "remotes", "upload", "tasks", "superproductivity", "caldav", "sites", "fetch", + "telemetry", "remotes", "intake", "tasks", "superproductivity", "caldav", "sites", "fetch", "backup", ] @@ -126,24 +126,30 @@ def _swap(monkeypatch, attr, shape): ) -def test_upload_follows_a_field_made_required_or_optional(root, monkeypatch): +def test_intake_follows_a_field_made_required(root, monkeypatch): from chemenu import upload - original = config_shapes.UPLOAD - write_settings(root, monkeypatch, upload=UPLOAD_OK) + original = config_shapes.INTAKE + write_settings(root, monkeypatch, intake=INTAKE_OK) assert upload.read_config(root).max_bytes == 1000 - _swap(monkeypatch, "UPLOAD", _add_required(original)) + _swap(monkeypatch, "INTAKE", _add_required(original)) with pytest.raises(ValidationError, match="added is missing"): upload.read_config(root) - _swap(monkeypatch, "UPLOAD", original) - (root / config.LOCAL_SETTINGS_FILENAME).unlink() - (root / config.INSTANCE_SETTINGS_FILENAME).unlink() - write_settings(root, monkeypatch, upload={k: v for k, v in UPLOAD_OK.items() if k != "max_bytes"}) - with pytest.raises(ValidationError): + +def test_intake_rules_across_fields_are_checked_by_the_record(root, monkeypatch): + from chemenu import upload + + write_settings(root, monkeypatch, intake={"channels": {"mcp": {"handler": "mcp", "max_bytes": 1}}}) + with pytest.raises(ValidationError, match="quota_submissions_per_day"): upload.read_config(root) - _swap(monkeypatch, "UPLOAD", _not_required(original, "max_bytes", 99)) - assert upload.read_config(root).max_bytes == 99 + + +@pytest.mark.parametrize("key", ["intake.channels", "intake.commands"]) +def test_the_intake_fields_are_not_settable_by_config_set(root, key): + result = runner.invoke(app, ["config", "set", key, "{}"]) + assert result.exit_code == 42, result.output + assert not (root / config.LOCAL_SETTINGS_FILENAME).exists() def test_tasks_follows_a_field_made_required(root, monkeypatch): diff --git a/tools/chemenu/tests/test_doctor.py b/tools/chemenu/tests/test_doctor.py index 0696939..0b0dcf2 100644 --- a/tools/chemenu/tests/test_doctor.py +++ b/tools/chemenu/tests/test_doctor.py @@ -350,27 +350,52 @@ def test_publish_remotes_warns_on_several_remotes_without_an_allowlist(instance) def test_upload_intake_ok_when_no_section(instance): checks = doctor.run_doctor() assert _status(checks, "upload-intake") == "OK" - assert "not registered" in _detail(checks, "upload-intake") + assert "no channel" in _detail(checks, "upload-intake") -def test_upload_intake_ok_and_armed_when_section_is_valid(instance, monkeypatch): - write_settings(config.ROOT, monkeypatch, upload={ - "max_bytes": 100, "allowed_extensions": [".md"], - "quota_submissions_per_day": 1, "quota_bytes_per_day": 100, - }) +_MCP = {"handler": "mcp", "max_bytes": 100, "allowed_extensions": [".md"], + "quota_submissions_per_day": 1, "quota_bytes_per_day": 100} + + +def test_upload_intake_names_each_channel_with_its_handler(instance, monkeypatch, tmp_path_factory): + folder = tmp_path_factory.mktemp("synced") + write_settings(config.ROOT, monkeypatch, intake={"channels": { + "mcp": _MCP, + "onedrive": {"handler": "folder", "path": str(folder), "max_bytes": 100, "allowed_extensions": [".md"]}, + }}) checks = doctor.run_doctor() assert _status(checks, "upload-intake") == "OK" detail = _detail(checks, "upload-intake") - assert "armed" in detail + assert "mcp (mcp)" in detail and "onedrive (folder)" in detail assert "0 submission(s) waiting" in detail +def test_upload_intake_warns_when_a_folder_does_not_exist(instance, monkeypatch, tmp_path_factory): + write_settings(config.ROOT, monkeypatch, intake={"channels": { + "onedrive": {"handler": "folder", "path": str(tmp_path_factory.mktemp("outside") / "gone"), "max_bytes": 100, + "allowed_extensions": [".md"]}, + }}) + checks = doctor.run_doctor() + assert _status(checks, "upload-intake") == "WARN" + assert "onedrive" in _detail(checks, "upload-intake") + + def test_upload_intake_fails_on_a_malformed_section(instance, monkeypatch): - write_settings(config.ROOT, monkeypatch, upload={"max_bytes": "many"}) + write_settings(config.ROOT, monkeypatch, intake={"channels": {"mcp": {"handler": "mcp", "max_bytes": "many"}}}) checks = doctor.run_doctor() assert _status(checks, "upload-intake") == "FAIL" +def test_upload_intake_fails_on_a_leftover_mcp_upload_directory(instance): + waiting = config.ROOT / "mcp-upload" / "sub-1" + waiting.mkdir(parents=True) + (waiting / "a.md").write_bytes(b"waiting") + checks = doctor.run_doctor() + assert _status(checks, "upload-intake") == "FAIL" + assert "mv mcp-upload quarantine" in _detail(checks, "upload-intake") + assert (waiting / "a.md").read_bytes() == b"waiting" + + def test_tasks_provider_ok_when_no_section(instance): checks = doctor.run_doctor() assert _status(checks, "tasks-provider") == "OK" diff --git a/tools/chemenu/tests/test_legacy_settings.py b/tools/chemenu/tests/test_legacy_settings.py index 0269de2..e4e2a6f 100644 --- a/tools/chemenu/tests/test_legacy_settings.py +++ b/tools/chemenu/tests/test_legacy_settings.py @@ -30,6 +30,9 @@ APP_PASSWORD = "sekrit-app-password-0815" THRESHOLDS = {"stalled_waiting_days": 14, "unpaged_project_weeks": 3, "someday_stale_months": 5} # name -> (old files by file name, what is in effect afterwards) +_MCP_CHANNEL = {"name": "mcp", "handler": "mcp", "path": None, "settle_seconds": 60, + "command_name": "", "command": ()} + CASES = { "telemetry-full": ( {".wikitool-telemetry.json": {"schema": 1, "enabled": False, "max_session_bytes": 1000, @@ -54,7 +57,7 @@ CASES = { "schema": 1, "identity_header": "X-Forwarded-User", "max_bytes": 2048, "allowed_extensions": [".md", ".txt"], "quota": {"submissions_per_day": 4, "bytes_per_day": 9000}}}, - {"upload": {"identity_header": "X-Forwarded-User", "max_bytes": 2048, + {"upload": {**_MCP_CHANNEL, "identity_header": "X-Forwarded-User", "max_bytes": 2048, "allowed_extensions": (".md", ".txt"), "submissions_per_day": 4, "bytes_per_day": 9000}}, ), @@ -185,7 +188,8 @@ def _expected(case: dict) -> dict: for key, value in case.items(): if key == "upload" and isinstance(value, str): value = { - "identity_header": _default("upload", "identity_header"), + **_MCP_CHANNEL, + "identity_header": upload.DEFAULT_IDENTITY_HEADER, "max_bytes": 100, "allowed_extensions": (".md",), "submissions_per_day": 1, "bytes_per_day": 5, } @@ -210,6 +214,17 @@ def test_every_shape_keeps_its_effective_value(checkout, case): assert _normal(_effective(checkout)) == _normal(_expected(expected)) +def test_the_old_upload_file_becomes_the_mcp_channel_of_the_intake_section(checkout): + _write_old(checkout, CASES["upload-full"][0]) + legacy_settings.migrate(checkout) + data = json.loads((checkout / config.LOCAL_SETTINGS_FILENAME).read_text(encoding="utf-8")) + assert "upload" not in data + assert data["intake"] == {"channels": {"mcp": { + "handler": "mcp", "identity_header": "X-Forwarded-User", "max_bytes": 2048, + "allowed_extensions": [".md", ".txt"], + "quota_submissions_per_day": 4, "quota_bytes_per_day": 9000}}} + + # --- b: only the tools file remains; no secret in the tree or the index ------- @@ -376,7 +391,7 @@ def test_the_telemetry_hook_neither_reads_nor_refuses_over_an_old_file(checkout) @pytest.mark.parametrize("key,value", [ - ("upload.max_bytes", "100"), + ("intake.channels", '{"mcp": {"handler": "mcp"}}'), ("remotes.allowed_push_urls", '["ssh://git@example.net/a.git"]'), ]) def test_config_set_on_a_gated_section_exits_42_and_writes_nothing(checkout, key, value): diff --git a/tools/chemenu/tests/test_mcp_server.py b/tools/chemenu/tests/test_mcp_server.py index 13ba7be..6687218 100644 --- a/tools/chemenu/tests/test_mcp_server.py +++ b/tools/chemenu/tests/test_mcp_server.py @@ -9,8 +9,8 @@ The acceptance criteria that are not about a value coming back: importable from the server - structural, not filtered; - every response carries the commit it was computed from; - telemetry cannot land inside the served checkout; -- the sixth tool, `submit`, exists only when the `upload` section of `.wikitool-local.json` opts a - checkout in, writes only under `mcp-upload/`, and never resolves an +- the sixth tool, `submit`, exists only when the `intake` section of `.wikitool-local.json` opts a + checkout in, writes only under `quarantine/`, and never resolves an identity from anywhere but the request's own headers. """ from __future__ import annotations @@ -44,13 +44,14 @@ from chemenu.mcp.server import ( # noqa: E402 - after the skip guard _UPLOAD_CFG = { "schema": 1, - "upload": { + "intake": {"channels": {"mcp": { + "handler": "mcp", "identity_header": "X-Forwarded-User", "max_bytes": 1024, "allowed_extensions": [".md"], "quota_submissions_per_day": 10, "quota_bytes_per_day": 100_000, - }, + }}}, } @@ -368,13 +369,38 @@ def test_submit_is_present_once_armed(corpus): assert names == {"search", "types", "describe_type", "lint", "status", "submit"} +def test_submit_is_absent_when_the_section_has_only_pulled_channels(corpus, tmp_path_factory): + folder = tmp_path_factory.mktemp("synced") + section = {"schema": 1, "intake": {"channels": {"onedrive": { + "handler": "folder", "path": str(folder), "max_bytes": 1024, "allowed_extensions": [".md"]}}}} + (corpus / ".wikitool-local.json").write_text(json.dumps(section), encoding="utf-8") + names = {tool.name for tool in asyncio.run(build_server(corpus, check_trace=False).list_tools())} + assert "submit" not in names + + +def test_submit_carries_the_limits_of_the_mcp_channel(corpus): + _arm_upload(corpus) + tools = asyncio.run(build_server(corpus, check_trace=False).list_tools()) + submit = next(tool for tool in tools if tool.name == "submit") + assert "1024 bytes" in submit.description and ".md" in submit.description + + +def test_a_leftover_mcp_upload_directory_refuses_to_build(corpus): + _arm_upload(corpus) + (corpus / "mcp-upload" / "sub-1").mkdir(parents=True) + (corpus / "mcp-upload" / "sub-1" / "a.md").write_bytes(b"waiting") + with pytest.raises(ValidationError, match="mcp-upload"): + build_server(corpus, check_trace=False) + assert (corpus / "mcp-upload" / "sub-1" / "a.md").read_bytes() == b"waiting" + + def test_a_malformed_upload_config_refuses_to_build(corpus): (corpus / ".wikitool-local.json").write_text("{not json", encoding="utf-8") with pytest.raises(ValidationError): build_server(corpus, check_trace=False) -def test_submit_writes_only_into_mcp_upload_incoming_stays_untouched(corpus): +def test_submit_writes_only_into_the_quarantine_incoming_stays_untouched(corpus): _arm_upload(corpus) server = build_server(corpus, check_trace=False) before = _tree(corpus) @@ -391,15 +417,16 @@ def test_submit_writes_only_into_mcp_upload_incoming_stays_untouched(corpus): assert manifest["submitter"] == "alice" assert manifest["submitter_source"] == "X-Forwarded-User" + assert (manifest["channel"], manifest["handler"], manifest["origin"]) == ("mcp", "mcp", "mcp:submit") after = _tree(corpus) changed = {p for p in after if after.get(p) != before.get(p)} assert changed - assert all(p.startswith("mcp-upload/") for p in changed) + assert all(p.startswith("quarantine/") for p in changed) assert list((corpus / "incoming").iterdir()) == [] # git itself is untouched - the fixture's own committed tree (kb/, types/) - # is unchanged, and HEAD did not move. `mcp-upload/` and the config file + # is unchanged, and HEAD did not move. `quarantine/` and the config file # are new, untracked paths (this fixture carries no .gitignore of its # own, unlike the real repo), which `git status` reports and is exactly # what a real checkout's `.gitignore` would hide - not a write to git. diff --git a/tools/chemenu/tests/test_upload.py b/tools/chemenu/tests/test_upload.py index dd17d02..de18fc9 100644 --- a/tools/chemenu/tests/test_upload.py +++ b/tools/chemenu/tests/test_upload.py @@ -20,8 +20,10 @@ CONTENT = b"hello world, this is a test submission\n" CONTENT_B64 = base64.b64encode(CONTENT).decode("ascii") -def _cfg(**overrides) -> upload.UploadConfig: +def _cfg(**overrides) -> upload.Channel: defaults = dict( + name="mcp", + handler="mcp", identity_header="X-Forwarded-User", max_bytes=1024, allowed_extensions=(".md", ".pdf"), @@ -29,7 +31,7 @@ def _cfg(**overrides) -> upload.UploadConfig: bytes_per_day=10_000, ) defaults.update(overrides) - return upload.UploadConfig(**defaults) + return upload.Channel(**defaults) def _tree(root: Path) -> dict[str, bytes]: @@ -53,15 +55,21 @@ def test_sanitize_filename_accepts_a_bare_name(): assert upload.sanitize_filename("report.md") == "report.md" -# --- read_config ------------------------------------------------------------- +# --- read_config / read_channels ----------------------------------------------- def test_read_config_absent_is_none(tmp_path): assert upload.read_config(tmp_path) is None + assert upload.read_channels(tmp_path) == {} def _section(tmp_path, **fields): + channel = dict({"handler": "mcp"}, **fields) + _intake(tmp_path, {"channels": {"mcp": channel}}) + + +def _intake(tmp_path, body): (tmp_path / config.LOCAL_SETTINGS_FILENAME).write_text( - json.dumps({"schema": 1, "upload": fields}), encoding="utf-8") + json.dumps({"schema": 1, "intake": body}), encoding="utf-8") GOOD = {"max_bytes": 100, "allowed_extensions": [".md"], @@ -110,6 +118,7 @@ def test_read_config_valid_file_parses(tmp_path): _section(tmp_path, identity_header="X-User", max_bytes=2048, allowed_extensions=[".MD", ".pdf"], quota_submissions_per_day=3, quota_bytes_per_day=999) cfg = upload.read_config(tmp_path) + assert cfg.name == "mcp" and cfg.handler == "mcp" assert cfg.identity_header == "X-User" assert cfg.max_bytes == 2048 assert cfg.allowed_extensions == (".md", ".pdf") @@ -128,13 +137,70 @@ def test_read_config_refuses_a_file_of_an_earlier_version(tmp_path): upload.read_config(tmp_path) +def test_two_mcp_channels_are_an_error(tmp_path): + channel = dict({"handler": "mcp"}, **GOOD) + _intake(tmp_path, {"channels": {"a": channel, "b": channel}}) + with pytest.raises(ValidationError, match="mcp"): + upload.read_channels(tmp_path) + + +def test_a_field_that_does_not_match_the_handler_is_an_error(tmp_path, tmp_path_factory): + outside = tmp_path_factory.mktemp("outside") + channel = {"handler": "folder", "path": str(outside), "max_bytes": 100, "allowed_extensions": [".md"], + "identity_header": "X-User"} + _intake(tmp_path, {"channels": {"f": channel}}) + with pytest.raises(ValidationError, match="identity_header"): + upload.read_channels(tmp_path) + + +def test_a_command_channel_needs_a_commands_entry(tmp_path): + channel = {"handler": "command", "command": "scanner", "max_bytes": 100, "allowed_extensions": [".md"]} + _intake(tmp_path, {"channels": {"s": channel}}) + with pytest.raises(ValidationError, match="scanner"): + upload.read_channels(tmp_path) + + +def test_a_folder_path_must_be_absolute_and_outside_the_instance(tmp_path, tmp_path_factory): + base = {"handler": "folder", "max_bytes": 100, "allowed_extensions": [".md"]} + _intake(tmp_path, {"channels": {"f": dict(base, path="relative/dir")}}) + with pytest.raises(ValidationError, match="absolute"): + upload.read_channels(tmp_path) + inside = tmp_path / "drop" + _intake(tmp_path, {"channels": {"f": dict(base, path=str(inside))}}) + with pytest.raises(ValidationError, match="inside the instance"): + upload.read_channels(tmp_path) + outside = tmp_path_factory.mktemp("outside") + _intake(tmp_path, {"channels": {"f": dict(base, path=str(outside))}}) + channel = upload.read_channels(tmp_path)["f"] + assert channel.handler == "folder" and channel.path == outside + assert channel.settle_seconds == 60 + assert upload.read_config(tmp_path) is None + + +def test_a_leftover_mcp_upload_directory_is_refused_and_untouched(tmp_path): + legacy = tmp_path / "mcp-upload" / "sub-1" + legacy.mkdir(parents=True) + (legacy / "a.md").write_bytes(b"waiting") + before = _tree(tmp_path) + problem = upload.legacy_dir_problem(tmp_path) + assert "mv mcp-upload quarantine" in problem and "Move-Item" in problem + with pytest.raises(ValidationError, match="mcp-upload"): + upload.refuse_legacy_dir(tmp_path) + assert _tree(tmp_path) == before + + +def test_an_empty_mcp_upload_directory_is_no_problem(tmp_path): + (tmp_path / "mcp-upload").mkdir() + assert upload.legacy_dir_problem(tmp_path) is None + + # --- the write choke point ---------------------------------------------------- def test_write_primitive_refuses_a_relative_escape(tmp_path): with pytest.raises(ValidationError): upload._write_atomic_within(tmp_path, Path("../outside.txt"), b"x") assert not (tmp_path / "outside.txt").exists() - assert not (tmp_path / "mcp-upload").exists() + assert not (tmp_path / "quarantine").exists() def test_write_primitive_refuses_an_absolute_path(tmp_path): @@ -146,7 +212,7 @@ def test_write_primitive_refuses_an_absolute_path(tmp_path): def test_write_primitive_refuses_a_symlink_escape(tmp_path): outside = tmp_path / "outside" outside.mkdir() - upload_dir = tmp_path / "mcp-upload" + upload_dir = tmp_path / "quarantine" upload_dir.mkdir() (upload_dir / "escape").symlink_to(outside, target_is_directory=True) @@ -173,12 +239,12 @@ def test_submit_happy_path_writes_manifest_and_content(tmp_path): assert manifest["submitter_source"] == "X-Forwarded-User" assert manifest["size"] == len(CONTENT) - submission_dir = tmp_path / "mcp-upload" / manifest["id"] + submission_dir = tmp_path / "quarantine" / manifest["id"] assert (submission_dir / "report.md").read_bytes() == CONTENT stored_manifest = json.loads((submission_dir / "manifest.json").read_text()) assert stored_manifest == manifest - ledger_lines = (tmp_path / "mcp-upload" / "ledger.jsonl").read_text().splitlines() + ledger_lines = (tmp_path / "quarantine" / "ledger.jsonl").read_text().splitlines() assert len(ledger_lines) == 1 event = json.loads(ledger_lines[0]) assert event["event"] == "submitted" @@ -342,10 +408,10 @@ def test_promote_moves_file_deletes_dir_and_ledgers_accepted(tmp_path): assert dest == tmp_path / "incoming" / "a.md" assert dest.read_bytes() == CONTENT - assert not (tmp_path / "mcp-upload" / manifest["id"]).exists() + assert not (tmp_path / "quarantine" / manifest["id"]).exists() events = [json.loads(line) for line in - (tmp_path / "mcp-upload" / "ledger.jsonl").read_text().splitlines()] + (tmp_path / "quarantine" / "ledger.jsonl").read_text().splitlines()] assert [e["event"] for e in events] == ["submitted", "accepted"] # nothing else in the tree changed @@ -363,7 +429,7 @@ def test_promote_refuses_when_the_file_is_missing_from_disk(tmp_path): defense in depth, since nothing in this module deletes a content file without its manifest.""" manifest = upload.submit(tmp_path, _cfg(), filename="a.md", content_b64=CONTENT_B64, submitter="t") - (tmp_path / "mcp-upload" / manifest["id"] / "a.md").unlink() + (tmp_path / "quarantine" / manifest["id"] / "a.md").unlink() with pytest.raises(ValidationError): upload.promote(tmp_path, manifest["id"]) @@ -378,7 +444,7 @@ def test_promote_refuses_when_incoming_already_has_the_name(tmp_path): upload.promote(tmp_path, manifest["id"]) # nothing moved: quarantine still holds it, incoming/ untouched - assert (tmp_path / "mcp-upload" / manifest["id"] / "a.md").read_bytes() == CONTENT + assert (tmp_path / "quarantine" / manifest["id"] / "a.md").read_bytes() == CONTENT assert (tmp_path / "incoming" / "a.md").read_text() == "already here" @@ -388,9 +454,9 @@ def test_reject_deletes_material_and_ledgers_with_reason(tmp_path): manifest = upload.submit(tmp_path, _cfg(), filename="a.md", content_b64=CONTENT_B64, submitter="t") upload.reject(tmp_path, manifest["id"], "license unclear") - assert not (tmp_path / "mcp-upload" / manifest["id"]).exists() + assert not (tmp_path / "quarantine" / manifest["id"]).exists() events = [json.loads(line) for line in - (tmp_path / "mcp-upload" / "ledger.jsonl").read_text().splitlines()] + (tmp_path / "quarantine" / "ledger.jsonl").read_text().splitlines()] assert events[-1]["event"] == "rejected" assert events[-1]["reason"] == "license unclear" assert events[-1]["sha256"] == manifest["sha256"] @@ -401,7 +467,7 @@ def test_reject_requires_a_reason(tmp_path): with pytest.raises(ValidationError): upload.reject(tmp_path, manifest["id"], "") # nothing deleted - assert (tmp_path / "mcp-upload" / manifest["id"]).exists() + assert (tmp_path / "quarantine" / manifest["id"]).exists() def test_reject_unknown_id_is_rejected(tmp_path): diff --git a/tools/chemenu/tests/test_upload_cmd.py b/tools/chemenu/tests/test_upload_cmd.py index 9e62a54..e47c23d 100644 --- a/tools/chemenu/tests/test_upload_cmd.py +++ b/tools/chemenu/tests/test_upload_cmd.py @@ -20,7 +20,9 @@ from chemenu.commands.upload_cmd import ( CONTENT = b"a test submission for the CLI layer\n" CONTENT_B64 = base64.b64encode(CONTENT).decode("ascii") -_CFG = upload.UploadConfig( +_CFG = upload.Channel( + name="mcp", + handler="mcp", identity_header="X-Forwarded-User", max_bytes=1024, allowed_extensions=(".md",), @@ -50,7 +52,7 @@ def test_accept_without_a_token_needs_clearance(root, capsys): assert manifest["sha256"] in out assert "--confirm" in out # nothing moved - assert (root / "mcp-upload" / manifest["id"]).exists() + assert (root / "quarantine" / manifest["id"]).exists() assert not (root / "incoming" / manifest["filename"]).exists() @@ -59,7 +61,7 @@ def test_accept_with_a_stale_token_needs_clearance_again(root): with pytest.raises(typer.Exit) as excinfo: upload_accept_command(submission_id=manifest["id"], confirm="not-the-real-token") assert excinfo.value.exit_code == EXIT_NEEDS_CLEARANCE - assert (root / "mcp-upload" / manifest["id"]).exists() + assert (root / "quarantine" / manifest["id"]).exists() def test_accept_with_the_right_token_promotes(root): @@ -67,13 +69,13 @@ def test_accept_with_the_right_token_promotes(root): token = upload.confirm_token(manifest) upload_accept_command(submission_id=manifest["id"], confirm=token) assert (root / "incoming" / manifest["filename"]).read_bytes() == CONTENT - assert not (root / "mcp-upload" / manifest["id"]).exists() + assert not (root / "quarantine" / manifest["id"]).exists() def test_reject_deletes_and_needs_no_gate(root): manifest = _seed(root) upload_reject_command(submission_id=manifest["id"], reason="license unclear") - assert not (root / "mcp-upload" / manifest["id"]).exists() + assert not (root / "quarantine" / manifest["id"]).exists() def test_reject_without_a_reason_is_a_validation_error(root): @@ -81,7 +83,7 @@ def test_reject_without_a_reason_is_a_validation_error(root): with pytest.raises(typer.Exit) as excinfo: upload_reject_command(submission_id=manifest["id"], reason="") assert excinfo.value.exit_code == 1 - assert (root / "mcp-upload" / manifest["id"]).exists() + assert (root / "quarantine" / manifest["id"]).exists() def test_list_json_matches_pending_manifests(root, capsys): diff --git a/tools/chemenu/upload.py b/tools/chemenu/upload.py index 3e28278..1fa3c64 100644 --- a/tools/chemenu/upload.py +++ b/tools/chemenu/upload.py @@ -1,5 +1,5 @@ -"""The MCP write path's one choke point: `mcp-upload/`, and nothing else -(Gitea #32). +"""The quarantine every intake channel ends in: `quarantine/`, and nothing else +(Gitea #32, #213). `#19`'s MCP read server has an absence property - nothing under `chemenu.commands` is importable from it, so a write function does not exist @@ -8,29 +8,34 @@ being true by itself: something in the server process now writes. What replaces it is not an absence but a **positive list**, enforced in code rather than promised in prose: - The server process may write into exactly one directory - `mcp-upload/` + The server process may write into exactly one directory - `quarantine/` under the served root - and every write in this module resolves its target and refuses anything that lands outside it. `_write_atomic_within()` is that one choke point: every file this module writes (a submitted file, its manifest) goes through it. The ledger append is the one exception, and it is a narrower case of the same rule rather than a -gap in it - its destination is a hardcoded constant (`mcp-upload/ledger.jsonl`), +gap in it - its destination is a hardcoded constant (`quarantine/ledger.jsonl`), never a caller-supplied name, so there is no path to sanitise in the first place. -Two stages, two different grants of trust: +Several channels (the `intake` section) end in the same place. A channel has +a handler: `mcp` is the `submit` tool, which pushes; `folder` and `command` +are pulled by `wikitool upload collect` (`chemenu.collect`), which reads a +folder or runs a program and hands each file to `store()` here. Nothing a +channel delivers reaches `incoming/` or `raw/` any other way: - mcp-upload// material nobody has looked at - `submit` writes here + quarantine// material nobody has looked at - every channel writes here | wikitool upload accept <- a human decides (Exit 42 gate) incoming/ the ordinary local intake (Gitea #58/#67) | wikitool raw accept ... raw/ `upload accept`/`upload reject` (the reviewer commands, `commands/upload_cmd.py`) -are **not** importable from here or from `chemenu.mcp.server` - they live under -`chemenu.commands`, structurally unreachable from the server, same as every -other write command #19 already keeps out. +and everything that reads outside the quarantine or starts a program +(`chemenu.collect`) are **not** importable from here or from +`chemenu.mcp.server` - they live outside it, structurally unreachable from the +server, same as every other write command #19 already keeps out. Stdlib only, like `chemenu.telemetry` - this module is imported by the MCP server process on every request, not only at CLI dispatch. @@ -47,12 +52,14 @@ import secrets import shutil from dataclasses import dataclass from pathlib import Path -from typing import Any, Optional +from typing import Any, Callable, Optional from chemenu import config, config_shapes, legacy_settings, settings from chemenu.errors import ValidationError -DEFAULT_IDENTITY_HEADER = config_shapes.UPLOAD.field("identity_header").default +DEFAULT_IDENTITY_HEADER = config_shapes.INTAKE_DEFAULT_IDENTITY_HEADER +GITIGNORE_NAME = ".gitignore" +STAGING_DIRNAME = ".staging" # Submissions older than this stop counting toward a submitter's quota. A # rolling window rather than a calendar day - "resets at midnight" is a @@ -62,22 +69,48 @@ _QUOTA_WINDOW = datetime.timedelta(days=1) _FORBIDDEN_FILENAME_CHARS = ("/", "\\", "\x00") +UNKNOWN_SUBMITTER = "unknown" + @dataclass(frozen=True) -class UploadConfig: - """The opt-in, read from the `upload` section of `.wikitool-local.json`.""" +class Channel: + """One channel of the `intake` section of `.wikitool-local.json`. Only the + fields of its handler are meaningful; the others keep their defaults.""" - identity_header: str + name: str + handler: str max_bytes: int allowed_extensions: tuple[str, ...] - submissions_per_day: int - bytes_per_day: int + identity_header: str = DEFAULT_IDENTITY_HEADER + submissions_per_day: int = 0 + bytes_per_day: int = 0 + path: Optional[Path] = None + settle_seconds: int = config_shapes.INTAKE_SETTLE_SECONDS + command_name: str = "" + command: tuple[str, ...] = () + + @property + def pulled(self) -> bool: + return self.handler != config_shapes.INTAKE_HANDLER_MCP -def read_config(root: "Path | str") -> Optional[UploadConfig]: - """The upload opt-in for `root`, or `None` when the `upload` section is not - set - which means the write path does not exist, not that it is - unrestricted. +def _folder_problem(name: str, raw: str, root: "Path | str") -> Optional[str]: + if not os.path.isabs(raw): + return f"intake.channels.{name}.path {raw!r} is not an absolute path on this machine" + real_root = Path(root).resolve() + resolved = Path(raw).resolve() + try: + resolved.relative_to(real_root) + except ValueError: + return None + return (f"intake.channels.{name}.path {raw!r} lies inside the instance - a folder the channel " + "empties belongs outside it") + + +def read_channels(root: "Path | str") -> dict[str, Channel]: + """Every channel of the `intake` section for `root`, by name, in the order + the file lists them - empty when the section is not set, which means no + channel exists, not that one is unrestricted. A malformed section is a `ValidationError`, never a silent "no limits": it decides whether an unauthenticated write path is offered at all, so a @@ -89,17 +122,79 @@ def read_config(root: "Path | str") -> Optional[UploadConfig]: does not exist. The file of an earlier version that is still there raises `LegacySettingsLeftover` rather than reading as "not configured". """ - legacy_settings.refuse_leftover(root, "upload") - section = settings.load(root, only=(config_shapes.UPLOAD.key,)).section(config_shapes.UPLOAD.key) + legacy_settings.refuse_leftover(root, "intake") + section = settings.load(root, only=(config_shapes.INTAKE.key,)).section(config_shapes.INTAKE.key) if not section.configured: + return {} + commands = section["commands"] + channels: dict[str, Channel] = {} + problems: list[str] = [] + for name, raw in section["channels"].items(): + handler = raw["handler"] + common = dict( + name=name, handler=handler, max_bytes=raw["max_bytes"], + allowed_extensions=config_shapes.extensions(raw["allowed_extensions"]), + ) + if handler == config_shapes.INTAKE_HANDLER_MCP: + channels[name] = Channel( + identity_header=raw.get("identity_header", DEFAULT_IDENTITY_HEADER), + submissions_per_day=raw["quota_submissions_per_day"], + bytes_per_day=raw["quota_bytes_per_day"], **common) + elif handler == config_shapes.INTAKE_HANDLER_FOLDER: + problem = _folder_problem(name, raw["path"], root) + if problem: + problems.append(problem) + continue + channels[name] = Channel( + path=Path(raw["path"]), + settle_seconds=raw.get("settle_seconds", config_shapes.INTAKE_SETTLE_SECONDS), **common) + else: + channels[name] = Channel( + command_name=raw["command"], command=tuple(commands[raw["command"]]), **common) + if problems: + raise ValidationError("; ".join(problems)) + return channels + + +def mcp_channel(channels: dict[str, Channel]) -> Optional[Channel]: + """The channel with the handler `mcp` - at most one exists - or `None`.""" + for channel in channels.values(): + if channel.handler == config_shapes.INTAKE_HANDLER_MCP: + return channel + return None + + +def read_config(root: "Path | str") -> Optional[Channel]: + """The `mcp` channel for `root`, or `None` when there is none - the `submit` + tool does not exist then. A malformed section raises like `read_channels`.""" + return mcp_channel(read_channels(root)) + + +def legacy_dir_problem(root: "Path | str") -> Optional[str]: + """What to tell the operator when the quarantine of earlier versions + (`mcp-upload/`) still holds something, or `None`. Nothing is moved for + them: material that waits for review is not ours to relocate.""" + legacy = Path(root) / config.LEGACY_UPLOAD_DIRNAME + try: + if not legacy.is_dir() or not any(legacy.iterdir()): + return None + except OSError: return None - return UploadConfig( - identity_header=section["identity_header"], - max_bytes=section["max_bytes"], - allowed_extensions=section["allowed_extensions"], - submissions_per_day=section["quota_submissions_per_day"], - bytes_per_day=section["quota_bytes_per_day"], - ) + new = config.UPLOAD_DIRNAME + if (Path(root) / new).exists(): + how = (f"{new}/ exists already, so move the entries of {config.LEGACY_UPLOAD_DIRNAME}/ into it " + f"one by one and delete the empty {config.LEGACY_UPLOAD_DIRNAME}/ afterwards") + else: + how = (f"POSIX: `mv {config.LEGACY_UPLOAD_DIRNAME} {new}`; " + f"pwsh: `Move-Item {config.LEGACY_UPLOAD_DIRNAME} {new}` - run in the instance root") + return (f"{config.LEGACY_UPLOAD_DIRNAME}/ still holds submissions from an earlier version, and the " + f"quarantine is {new}/ now ({how}). Nothing was changed.") + + +def refuse_legacy_dir(root: "Path | str") -> None: + problem = legacy_dir_problem(root) + if problem: + raise ValidationError(problem) def sanitize_filename(name: str) -> str: @@ -153,12 +248,26 @@ def _parse_iso(value: Any) -> Optional[datetime.datetime]: def _upload_root(root: "Path | str") -> Path: - return (Path(root) / "mcp-upload").resolve() + return (Path(root) / config.UPLOAD_DIRNAME).resolve() + + +def _ensure_quarantine(root: "Path | str") -> Path: + """The quarantine directory, created with a `.gitignore` of its own that + ignores everything in it - so material nobody has looked at is never + committed, not even from a checkout whose root `.gitignore` predates the + directory's name. The file is a constant written in place, never a + caller-supplied name.""" + base = _upload_root(root) + base.mkdir(parents=True, exist_ok=True) + marker = base / GITIGNORE_NAME + if not marker.exists(): + marker.write_text("*\n", encoding="utf-8", newline="\n") + return base def _write_atomic_within(root: "Path | str", relative: Path, data: bytes) -> Path: """The one choke point every content/manifest write in this module goes - through: resolve the target, refuse anything outside `mcp-upload/` - a + through: resolve the target, refuse anything outside `quarantine/` - a resolved path also closes a symlink or a `..` in `relative` - then write it via a temp file plus `os.replace` so a reader never observes a partial file.""" @@ -167,7 +276,8 @@ def _write_atomic_within(root: "Path | str", relative: Path, data: bytes) -> Pat try: target.relative_to(base) except ValueError: - raise ValidationError(f"refusing to write outside mcp-upload/: {relative}") + raise ValidationError(f"refusing to write outside {config.UPLOAD_DIRNAME}/: {relative}") + _ensure_quarantine(root) target.parent.mkdir(parents=True, exist_ok=True) tmp = target.with_name(f".{target.name}.{secrets.token_hex(4)}.tmp") tmp.write_bytes(data) @@ -177,7 +287,7 @@ def _write_atomic_within(root: "Path | str", relative: Path, data: bytes) -> Pat def _pending_dir(root: "Path | str", submission_id: str) -> Path: """The quarantine directory for `submission_id`, refusing a `submission_id` - that would resolve outside `mcp-upload/` - this one comes from a CLI + that would resolve outside `quarantine/` - this one comes from a CLI argument, a human, not the generator above, so it gets the same containment check as a write.""" base = _upload_root(root) @@ -190,12 +300,11 @@ def _pending_dir(root: "Path | str", submission_id: str) -> Path: def _append_ledger(root: "Path | str", event: dict) -> None: - """Append one event to `mcp-upload/ledger.jsonl` - the destination is this + """Append one event to `quarantine/ledger.jsonl` - the destination is this literal constant, never a caller-supplied path, which is what makes an append-mode write (rather than `_write_atomic_within`'s replace) safe: there is nothing here for untrusted input to redirect.""" - base = _upload_root(root) - base.mkdir(parents=True, exist_ok=True) + base = _ensure_quarantine(root) path = base / "ledger.jsonl" line = json.dumps(event, sort_keys=True) + "\n" with path.open("a", encoding="utf-8", newline="\n") as fh: @@ -238,7 +347,7 @@ def _pending_manifests(root: "Path | str") -> list[dict]: def digest_to_pending_id(root: "Path | str", digest: str) -> Optional[str]: """The id of a submission already waiting with this exact sha256, if any. Only *pending* submissions are consulted - an accepted or rejected one no - longer has a manifest under `mcp-upload/` - so a re-submission after a + longer has a manifest under `quarantine/` - so a re-submission after a rejection is not blocked by this check.""" for manifest in _pending_manifests(root): if manifest.get("sha256") == digest: @@ -246,13 +355,15 @@ def digest_to_pending_id(root: "Path | str", digest: str) -> Optional[str]: return None -def _check_quota(root: "Path | str", cfg: UploadConfig, submitter: str, size: int) -> None: +def _check_quota(root: "Path | str", cfg: Channel, submitter: str, size: int) -> None: window_start = datetime.datetime.now(datetime.timezone.utc) - _QUOTA_WINDOW count = 0 total_bytes = 0 for event in _read_ledger(root): if event.get("event") != "submitted" or event.get("submitter") != submitter: continue + if event.get("handler", config_shapes.INTAKE_HANDLER_MCP) != config_shapes.INTAKE_HANDLER_MCP: + continue # a pulled channel has no submitter, and no quota when = _parse_iso(event.get("time")) if when is None or when < window_start: continue @@ -270,9 +381,73 @@ def _check_quota(root: "Path | str", cfg: UploadConfig, submitter: str, size: in ) +class SourceChanged(Exception): + """The `verify` hook of `store()` found that the source no longer matches + what was read - nothing of the submission was kept.""" + + +def store( + root: "Path | str", + channel: Channel, + *, + filename: str, + data: bytes, + submitter: str, + submitter_source: str, + origin: str, + verify: Optional[Callable[[], bool]] = None, +) -> dict: + """Write one already-checked submission into the quarantine - its file, its + manifest, a `submitted` ledger event - and return the manifest. + + Every channel ends here, so the manifest says where the file came from: + `channel`, `handler` and `origin` (the source file's path, `command:` + or `mcp:submit`). `verify`, when given, runs after the file is fully + written and before anything marks the submission as existing: if it + returns false the half submission is removed and `SourceChanged` is + raised, so a source that moved under the copy is never kept as if it + had been read cleanly.""" + clean_name = sanitize_filename(filename) + digest = hashlib.sha256(data).hexdigest() + submission_id = new_submission_id() + _write_atomic_within(root, Path(submission_id) / clean_name, data) + if verify is not None and not verify(): + shutil.rmtree(_pending_dir(root, submission_id), ignore_errors=True) + raise SourceChanged(clean_name) + manifest = { + "schema": 1, + "id": submission_id, + "filename": clean_name, + "size": len(data), + "sha256": digest, + "submitter": submitter, + "submitter_source": submitter_source, + "submitted_at": _now_iso(), + "channel": channel.name, + "handler": channel.handler, + "origin": origin, + } + _write_atomic_within( + root, + Path(submission_id) / "manifest.json", + (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode("utf-8"), + ) + _append_ledger(root, { + "event": "submitted", + "id": submission_id, + "channel": channel.name, + "handler": channel.handler, + "submitter": submitter, + "time": manifest["submitted_at"], + "size": len(data), + "sha256": digest, + }) + return manifest + + def submit( root: "Path | str", - cfg: UploadConfig, + cfg: Channel, *, filename: str, content_b64: str, @@ -338,32 +513,31 @@ def submit( _check_quota(root, cfg, submitter, len(data)) - submission_id = new_submission_id() - _write_atomic_within(root, Path(submission_id) / clean_name, data) - manifest = { - "schema": 1, - "id": submission_id, - "filename": clean_name, - "size": len(data), - "sha256": digest, - "submitter": submitter, - "submitter_source": cfg.identity_header, - "submitted_at": _now_iso(), - } - _write_atomic_within( - root, - Path(submission_id) / "manifest.json", - (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode("utf-8"), + return store( + root, cfg, filename=clean_name, data=data, submitter=submitter, + submitter_source=cfg.identity_header, origin="mcp:submit", ) - _append_ledger(root, { - "event": "submitted", - "id": submission_id, - "submitter": submitter, - "time": manifest["submitted_at"], - "size": len(data), - "sha256": digest, - }) - return manifest + + +def staging_dir(root: "Path | str") -> Path: + """A new, empty directory for a `command` channel's program to fill, under + `quarantine/.staging/` - inside the quarantine, so a program that writes + nowhere else leaves nothing outside it. The caller removes it.""" + base = _ensure_quarantine(root) + parent = base / STAGING_DIRNAME + parent.mkdir(exist_ok=True) + path = parent / secrets.token_hex(8) + path.mkdir() + return path + + +def drop_staging(root: "Path | str", path: Path) -> None: + """Remove a staging directory and, once nothing else is staged, `.staging/`.""" + shutil.rmtree(path, ignore_errors=True) + try: + (_upload_root(root) / STAGING_DIRNAME).rmdir() + except OSError: + pass def list_submissions(root: "Path | str") -> list[dict]: @@ -375,7 +549,7 @@ def list_submissions(root: "Path | str") -> list[dict]: def read_manifest(root: "Path | str", submission_id: str) -> dict: path = _pending_dir(root, submission_id) / "manifest.json" if not path.is_file(): - raise ValidationError(f"No submission '{submission_id}' is waiting in mcp-upload/.") + raise ValidationError(f"No submission '{submission_id}' is waiting in quarantine/.") try: data = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: @@ -430,6 +604,8 @@ def promote(root: "Path | str", submission_id: str) -> Path: _append_ledger(root, { "event": "accepted", "id": submission_id, + "channel": manifest.get("channel"), + "handler": manifest.get("handler"), "submitter": manifest.get("submitter"), "time": _now_iso(), "size": manifest.get("size"), @@ -449,6 +625,8 @@ def reject(root: "Path | str", submission_id: str, reason: str) -> None: _append_ledger(root, { "event": "rejected", "id": submission_id, + "channel": manifest.get("channel"), + "handler": manifest.get("handler"), "submitter": manifest.get("submitter"), "time": _now_iso(), "size": manifest.get("size"),