From 9736472f485333cfd08493da3e26c371f89f1f39 Mon Sep 17 00:00:00 2001 From: Torben Nehmer Date: Mon, 12 Oct 2026 19:39:49 +0200 Subject: [PATCH] backup: an instance backs up into a synced folder - backup init makes a bare repository under /git its origin, backup run and publish mirror the machine data into //, doctor warns about .git in a synced folder (#186); 10.0.0-beta.10 Files changed: - CHANGES.md - INSTALL.md - VERSION - docs/explanation/configuration-tiers.md - docs/how-to/README.md - docs/how-to/back-up-to-a-synced-folder.md - docs/reference/commands/README.md - docs/reference/commands/git.md - docs/reference/commands/instance-health.md - docs/reference/configuration.md - instructions/publish-cycle.md - instructions/setup-instance.md - tools/CONTRACT.md - tools/README.md - tools/chemenu/backup.py - tools/chemenu/cli.py - tools/chemenu/cli_contract.py - tools/chemenu/commands/backup_cmd.py - tools/chemenu/commands/doctor.py - tools/chemenu/commands/git_publish.py - tools/chemenu/config_shapes.py - tools/chemenu/reference_doc.py - tools/chemenu/tests/conftest.py - tools/chemenu/tests/test_backup.py - tools/chemenu/tests/test_config_shapes.py Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2 --- CHANGES.md | 7 +- INSTALL.md | 4 +- VERSION | 2 +- docs/explanation/configuration-tiers.md | 17 + docs/how-to/README.md | 2 + docs/how-to/back-up-to-a-synced-folder.md | 158 ++++++ docs/reference/commands/README.md | 2 + docs/reference/commands/git.md | 108 +++- docs/reference/commands/instance-health.md | 3 + docs/reference/configuration.md | 53 +- instructions/publish-cycle.md | 7 +- instructions/setup-instance.md | 8 + tools/CONTRACT.md | 116 ++++- tools/README.md | 1 + tools/chemenu/backup.py | 563 +++++++++++++++++++++ tools/chemenu/cli.py | 2 + tools/chemenu/cli_contract.py | 2 +- tools/chemenu/commands/backup_cmd.py | 169 +++++++ tools/chemenu/commands/doctor.py | 87 +++- tools/chemenu/commands/git_publish.py | 66 ++- tools/chemenu/config_shapes.py | 94 +++- tools/chemenu/reference_doc.py | 1 + tools/chemenu/tests/conftest.py | 4 + tools/chemenu/tests/test_backup.py | 562 ++++++++++++++++++++ tools/chemenu/tests/test_config_shapes.py | 1 + 25 files changed, 2024 insertions(+), 15 deletions(-) create mode 100644 docs/how-to/back-up-to-a-synced-folder.md create mode 100644 tools/chemenu/backup.py create mode 100644 tools/chemenu/commands/backup_cmd.py create mode 100644 tools/chemenu/tests/test_backup.py diff --git a/CHANGES.md b/CHANGES.md index f353296..adc7a0d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,7 +14,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. --- -## 10.0.0-beta.9 - 2026-10-11 - Gesprächssprache: AGENTS.md § Conversation language legt fest, dass der Agent im Chat die KB-Sprache spricht; instructions sync schreibt den Sprachwert als generierte Zeile in jedes veröffentlichte SKILL.md, instructions verify und doctor melden eine geänderte Sprache als Drift +## 10.0.0-beta.10 - 2026-10-12 - Backup in einen Sync-Ordner (OneDrive): backup init legt ein Bare-Repo unter /git als origin an, backup run und publish spiegeln die Maschinendaten nach //, doctor warnt vor .git im Sync-Ordner **Author:** Torben Nehmer @@ -33,6 +33,7 @@ place only: `instructions/dev/version-parts.md` § The candidate model. - raw fetch --scope: eine Seite und die Unterseiten, die ihr Site-Handler anbietet, als ein Bündel in incoming/ - raw fetch hinter einem Login: raw login speichert die Session des Nutzers als Secret, raw fetch holt damit; gerenderte Seiten über die optionale Browser-Komponente, externe Abrufer pro Host - Gesprächssprache: AGENTS.md § Conversation language legt fest, dass der Agent im Chat die KB-Sprache spricht; instructions sync schreibt den Sprachwert als generierte Zeile in jedes veröffentlichte SKILL.md, instructions verify und doctor melden eine geänderte Sprache als Drift +- Backup in einen Sync-Ordner (OneDrive): backup init legt ein Bare-Repo unter /git als origin an, backup run und publish spiegeln die Maschinendaten nach //, doctor warnt vor .git im Sync-Ordner **Low impact** - Ketten von Paketen: stack-dev schreibt ## Kette, stack-close nennt das nächste Glied @@ -62,6 +63,10 @@ Der Abschnitt `fetch` der Einstellungen ordnet einem Host einen Abrufweg zu. Bei Die vier Einzeldateien `.wikitool-remotes.json`, `.wikitool-telemetry.json`, `.wikitool-upload.json` und `.wikitool-tasks.json` gibt es nicht mehr; ihre Felder sind die Abschnitte `remotes`, `telemetry`, `upload`, `tasks`, `superproductivity` und `caldav` der drei Stufen. Zugangsdaten sind dort nur noch Verweise. Der Code liest die alten Dateien nicht mehr: ein Kommando, das die Einstellung braucht, endet mit Exit 42 und nennt `tools/wikitool config migrate`, `doctor` meldet FAIL, der Telemetrie-Hook bleibt still. `config migrate` läuft einmal je Checkout, ist wiederholbar und lässt bei jedem Abbruch die alten Dateien unverändert. `WIKITOOL_TASKS_CONFIG` zeigt auf ein Profil im Format von `.wikitool-local.json`. Nur `.wikitool-tools.json` bleibt eine eigene Datei. +### Backup in einen Sync-Ordner (#186) + +Ohne Git-Server bleibt der Arbeitsbaum außerhalb des Sync-Ordners; gesichert wird in das Ziel, das die Sektion `backup` von `.wikitool-local.json` nennt. `backup init` legt `/git` als Bare-Repo an und macht es zu `origin`, wenn keins existiert. Mit `machine_data: true` spiegelt jedes `publish` nach dem Push `.wikitool-local.json`, `ENVIRONMENT.md`, `incoming/` und die `extra`-Pfade nach `//` samt Manifest; `backup run` tut dasselbe von Hand. Geschrieben wird nur dort und atomar, gelesen nur innerhalb der Instanz, Secrets nie. Eine gescheiterte Sicherung nach dem Push ist eine `WARN`-Zeile mit Exit 0, eine fehlerhafte Sektion stoppt `publish` vor dem Commit. `doctor` meldet `sync-folder` und `backup`. Ablauf und Wiederherstellung: `docs/how-to/back-up-to-a-synced-folder.md`. + --- ## 9.0.0 - 2026-10-09 - Docs sorted into four areas, generated references, raw derive for real documents diff --git a/INSTALL.md b/INSTALL.md index 8187516..3108de5 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -100,7 +100,9 @@ repository: needed). - **Remote** - for an empty clone, only the confirmation that `origin` is right; otherwise a URL, if you want to push to a server. Without a remote the - instance stays local, and every `publish` runs with `--no-push`. + instance stays local, and every `publish` runs with `--no-push`. Without a git server but with a + synced folder (OneDrive and the like), the agent offers a backup into that folder instead: + [docs/how-to/back-up-to-a-synced-folder.md](docs/how-to/back-up-to-a-synced-folder.md). - **Language and tone of the pages** - they go into `kb/CONVENTIONS.md`, plus `kb//COLLECTION.md` per collection. Ready-made profiles, including a complete German one, are in `instructions/kb-profiles.md`. Decide this **before the diff --git a/VERSION b/VERSION index 3869d08..d7b4f75 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -10.0.0-beta.9 +10.0.0-beta.10 diff --git a/docs/explanation/configuration-tiers.md b/docs/explanation/configuration-tiers.md index 1e3f37f..b4b3a11 100644 --- a/docs/explanation/configuration-tiers.md +++ b/docs/explanation/configuration-tiers.md @@ -7,6 +7,16 @@ command that writes them are in [the configuration reference](../reference/confi the rules an agent follows are in [AGENTS.md § Settings](../../AGENTS.md#settings). This page is about why it is built this way, and what it does not do. + +## Contents + +- [Three kinds of data that used to share files](#three-kinds-of-data-that-used-to-share-files) +- [One place per value, not an override](#one-place-per-value-not-an-override) +- [What runs, and where it pushes, never travels with a pull](#what-runs-and-where-it-pushes-never-travels-with-a-pull) +- [References instead of values](#references-instead-of-values) +- [What this does not protect against](#what-this-does-not-protect-against) + + ## Three kinds of data that used to share files Before the tiers, every feature kept its settings in a gitignored file of its own. Three kinds of @@ -57,6 +67,13 @@ same value. In CI, `op` runs with a service-account token and resolves the same references. Alternatively, 1Password's own action resolves values into variables before `wikitool` starts, which serves references by name. This repository's CI does the second. +A plain private value in `.wikitool-local.json` stays on this machine only as long as that file +does. A backup into a synced folder (the `backup` section) copies the file as it is, because it is +the one record of how this checkout is set up, and a restore without it starts from nothing. That +trade is the operator's, made by writing the section by hand. A reference keeps the value out of +that copy as well, and the secrets file is never in it: it lies outside the instance, and a backup +reads nothing outside it. + A reference is resolved only when a command asks for that one value, never when the files are read, and the value stays in that process's memory. `doctor` reports whether each reference resolves, never what it resolves to. diff --git a/docs/how-to/README.md b/docs/how-to/README.md index 8674784..3281b15 100644 --- a/docs/how-to/README.md +++ b/docs/how-to/README.md @@ -30,5 +30,7 @@ Running the instance: - [Trace and score agent sessions](trace-and-score-sessions.md) - [Use 1Password for settings](use-1password-for-settings.md) - secrets and private values out of the tree, on the desktop and in CI +- [Back up to a synced folder](back-up-to-a-synced-folder.md) - OneDrive and the like as the + backup of an instance without a git server, and restoring from it Installing a new instance is [INSTALL.md](../../INSTALL.md) at the repository root. diff --git a/docs/how-to/back-up-to-a-synced-folder.md b/docs/how-to/back-up-to-a-synced-folder.md new file mode 100644 index 0000000..f11211a --- /dev/null +++ b/docs/how-to/back-up-to-a-synced-folder.md @@ -0,0 +1,158 @@ +# Back up to a synced folder + +Back an instance up into a folder a sync client uploads - OneDrive, or any other client that keeps a +local folder in step with a server - when there is no git server to push to. The instance itself +stays in a folder the sync client does not touch; the synced folder receives a git repository to +push to and a copy of the files on this machine that are not in git. What the settings look like: +[configuration reference](../reference/configuration.md#backup-to-a-synced-folder). + + +## Contents + +- [Why the instance does not live in the synced folder](#why-the-instance-does-not-live-in-the-synced-folder) +- [Set it up](#set-it-up) +- [What is backed up, and what is not](#what-is-backed-up-and-what-is-not) +- [Restore](#restore) +- [One device writes](#one-device-writes) + + +## Why the instance does not live in the synced folder + +Putting the whole instance - `.git` included - into the synced folder looks like the simplest +backup, and fails in three ways: + +- **Locks.** On Windows, OneDrive holds a file it is uploading open for a moment. `git gc` and + `git repack` then fail to delete or replace a file under `.git` ("unlink failed"). +- **Placeholders.** With Files On-Demand, the client offloads files it considers unused and leaves a + placeholder behind; git then reads objects that first have to be downloaded, or are not there + when the machine is offline. +- **Conflict copies.** As soon as a second device syncs the same folder, two writers meet inside + `.git`, and the client resolves that by making copies of the files both changed - a repository + git can no longer read. + +So the instance lives outside the synced folder, and the synced folder gets a layout git writes to +in a way a sync client handles well: + +``` +/git/ a bare repository - origin of the instance; publish pushes to it +// a copy of this machine's files that are not in git, plus .chemenu-backup.json +``` + +Git writes into the bare repository only on a push, and then mostly adds new files that never +change again. + +`tools/wikitool doctor` reports `sync-folder` as a `WARN` when the instance's git directory lies in +a synced folder. On Windows, check that line even if you never chose such a folder: when Documents +or Desktop is redirected into OneDrive (Known Folder Move), an instance under Documents lies in +OneDrive without looking like it. Move the instance to a folder outside it - `C:\Wiki\`, say - +before you go on. + +## Set it up + +1. **Pick the target**: a folder inside the synced folder, for example + `C:\Users\\OneDrive - \Chemenu-Backup`. It must not lie inside the instance. +2. **Write the `backup` section** into `.wikitool-local.json` at the root of the instance - by hand, + since its fields decide what leaves this machine and `config set` refuses them: + + ```json + { + "backup": { + "target": "C:\\Users\\\\OneDrive - \\Chemenu-Backup", + "machine_data": true, + "machine_name": "WORK-LAPTOP", + "extra": [".claude/settings.local.json"] + } + } + ``` + + Backslashes in a JSON string are doubled. `machine_name` defaults to the computer's host name; + `extra` lists further paths inside the instance to copy, and can be left out. +3. **Tell the agent:** + + > Set up the backup target from the `backup` section. + + The agent runs `tools/wikitool backup init`. It creates `/git` as a bare repository and + makes it `origin` - unless the instance already has an `origin` that points elsewhere: then it + changes nothing and shows you both URLs, and whether `origin` moves is yours to decide + (`git remote set-url origin "\git"`). +4. **Allow the push target**, if your `.wikitool-local.json` has a `remotes` section: `backup init` + then prints that section with `\git` added. Copy it into the file yourself - the list is + the Publish-Remote Gate, and no agent edits it. Without a `remotes` section there is nothing to do. +5. **Publish.** The next `tools/wikitool publish` pushes the whole history to `/git` and, + with `machine_data` on, copies the machine data into `//`. From then on every + publish that pushes does both. `tools/wikitool backup run` copies the machine data on its own, + whenever you want it current without publishing - after dropping a file into `incoming/`, for + instance, which is otherwise only copied by the next publish. +6. **Check it:** `tools/wikitool doctor`. The `backup` line names the target and how old the last + copy of the machine data is; a `WARN` there names what is missing. + +If a publish reports a `WARN` that the machine-data backup failed, the commit and the push have +still happened. Fix what the line names - a synced folder that is not there, a folder that is not +writable - and run `tools/wikitool backup run`. Another publish does not help: it has nothing new to +push. + +## What is backed up, and what is not + +| Path | Backed up | Why | +|------|-----------|-----| +| Everything committed (`kb/`, `raw/`, `wikitool.json`, ...) | yes, in `/git` | that is what `publish` pushes | +| `.wikitool-local.json` | yes | the settings of this checkout | +| `ENVIRONMENT.md` | yes, if present | written by hand | +| Everything in `incoming/` but `.gitkeep` | yes | sources waiting there are backed up nowhere else | +| The `extra` paths | yes | what you named | +| `mcp-upload/` | no | material submitted from outside that nobody has reviewed yet | +| `reports/` | no | regenerated by `lint`; telemetry holds prompts | +| `.wikitool-tools.json`, `tools/.venv/`, the published skills | no | the preflight and `instructions sync` rebuild them | +| The secrets file | **never** | it lies outside the instance, and a backup reads nothing outside it | + +**Private values reach the synced folder.** `.wikitool-local.json` is copied as it is, so a plain +private value in it - an internal host name, an internal URL - ends up in OneDrive. A secret does +not: a secret is never in that file, only a reference to it, and the secrets file it resolves +through (`%APPDATA%\chemenu\\secrets.json`, with the sessions `raw login` stored) +lies outside the instance. An `extra` path that leads outside the instance - through `..` or a +link - is refused, so no setting can copy it either. + +In `//` a run writes and deletes only what is its own: it removes a file only when +its previous run copied it and the instance no longer has it (a source `raw accept` moved out of +`incoming/`, say), and refuses a folder that is not empty and has no `.chemenu-backup.json`, rather +than copying into somebody else's folder. A link inside a copied directory is skipped and named, +never followed. + +## Restore + +On the same machine or a new one: + +1. **Clone the repository and check it:** + + ```bash + git clone "/git" "" + cd "" + git fsck --full + ``` + + The clone is outside the synced folder, like the original; its `origin` is the bare repository + already. +2. **Copy the machine data back**: everything in `//` except + `.chemenu-backup.json`, into the new folder, keeping the paths. On Windows: + `robocopy "\" "" /E /XF .chemenu-backup.json`; elsewhere: + `rsync -a --exclude .chemenu-backup.json "//" "/"`. +3. **Bootstrap the checkout** - tell the agent: + + > Bootstrap this checkout. + + It follows [instructions/bootstrap.md](../../instructions/bootstrap.md): the preflight, the + published skills, `doctor`. +4. **Provide the secrets again.** They were never in the backup. Put each value back where its + reference reads it - 1Password, the secrets file, or a `WIKITOOL_SECRET_` variable + ([use-1password-for-settings.md](use-1password-for-settings.md)) - and sign in again with + `raw login` for each host that needs it. `doctor`'s `settings-secrets` line names every reference + that does not resolve yet. + +`doctor` must then run through without a `FAIL`. + +## One device writes + +Only one device publishes to `/git`. Two devices pushing into the same synced folder are two +writers again, only one step removed - and the sync client resolves that with conflict copies, not +with a merge. An instance on a second device gets a target of its own. The sync client is a backup +here, not a way to work on one instance from two devices. diff --git a/docs/reference/commands/README.md b/docs/reference/commands/README.md index f5c8219..3ed7ab4 100644 --- a/docs/reference/commands/README.md +++ b/docs/reference/commands/README.md @@ -97,6 +97,8 @@ Full records: [git.md](git.md) - [`sync`](git.md#sync) - Fetch `/` and bring the local branch up to date with it. - [`publish`](git.md#publish) - Reconcile with `/`, then stage all changes, commit, and push. +- [`backup init`](git.md#backup-init) - Create the backup target's bare repository and make it `origin` when there is none. +- [`backup run`](git.md#backup-run) - Mirror this machine's files that are not in git into `//`. ## Guideline export diff --git a/docs/reference/commands/git.md b/docs/reference/commands/git.md index 2fdf2be..1d18c3f 100644 --- a/docs/reference/commands/git.md +++ b/docs/reference/commands/git.md @@ -1,12 +1,15 @@ # Git -Pulling in the remote and publishing a change - the only way a change leaves this checkout. +Pulling in the remote and publishing a change - the only way a change leaves this checkout - and +backing the checkout up into a synced folder. ## Contents - [`sync`](#sync) - [`publish`](#publish) +- [`backup init`](#backup-init) +- [`backup run`](#backup-run) @@ -130,6 +133,7 @@ tools/wikitool publish --confirm-rebase --message "ingest: docker-cheats - The remotes settings file of an earlier version is not read: while it exists this exits 42 and names `tools/wikitool config migrate`, so a restriction is never lost silently. - `--path` (repeatable) scopes the whole operation - gate count, staging and commit - to that subtree, with one exception: uncommitted generated files outside `--path` (`kb/index.md`, `kb/log.md`, `kb/provenance.md`, every `INDEX.md` under `kb/`) are staged and committed too, and a line names them - whether a reconcile or the change itself wrote them. They stay out of the gate count, and the `--confirm` token and the re-run line cover the `--path` given. Every other file outside `--path` is left as it is. - Commit message: `--message`, then a `Files changed:` paragraph listing every committed path. When `--message` ends in a paragraph git reads as a trailer block (`Co-Authored-By:` and the like), that block stays the last paragraph and the list goes in front of it, since git reads trailers from the last paragraph only. `git interpret-trailers` decides whether there is such a block, and the list moves only when git reads the same trailers from the result; otherwise it is appended at the end. `--message` is not part of any gate token. +- Machine-data backup: with a `backup` section in `.wikitool-local.json` whose `machine_data` is true, a call that pushed then mirrors this machine's files that are not in git into `//` - the same step as `backup run`. A failure there prints a `WARN` line with the reason and still exits 0, since the commit and the push stand. No backup with `--no-push`, after a failed push, or when there was nothing to push. A malformed `backup` section stops the call with exit 1 before anything is committed, `--no-push` included. - After a successful commit or push whose changed files include `tools/`, `types/`, `instructions/`, `AGENTS.md` or a path ending in `CONTRACT.md`, prints one reminder line: the phase past this point (an issue-body rewrite, `docs/` staleness, a changelog entry's accuracy) is not covered by `docs verify`, `instructions verify` or `pytest`. It is not a gate: no exit code change, nothing to clear, and silent for an ordinary content publish. **Properties** @@ -144,11 +148,12 @@ tools/wikitool publish --confirm-rebase --message "ingest: docker-cheats **Exit codes** - `0` - success +- `0` - The machine-data backup after the push failed: a `WARN` line names the reason, and the commit and the push stand - `1` - git failed - `git add`, `git commit`, `git push`, or the reconcile's automatic rebase (a conflict outside the generated files, an edited `kb/log.md`, or an uncommitted change on a path the incoming commits change) - `1` - The push target (`--branch`) is not the checked-out branch, or HEAD is detached; the unborn branch of a fresh `git init` is not this case - `1` - No `--no-push`, and the remote is not configured or cannot be reached; nothing was committed - `1` - `--yes`/`-y` was passed - the flag does not exist and fails with an explicit error -- `1` - The settings files do not pass their check, or `remotes.allowed_push_urls` is not a list of strings +- `1` - The settings files do not pass their check, `remotes.allowed_push_urls` is not a list of strings, or the `backup` section is malformed; nothing was committed - `42` - The remotes settings file of an earlier version is still there - `42` - Mass-Update Gate: `--threshold` (default 10) or more counted files would be committed, or the `--confirm` token does not match this changeset - `42` - Rebase-Review Gate: `/` moved and both sides changed the same file; the output lists the upstream commits, the overlapping files and their diff @@ -159,4 +164,103 @@ tools/wikitool publish --confirm-rebase --message "ingest: docker-cheats - `wikitool sync` - the same reconcile on its own, without committing or pushing - `instructions/gates.md` - the gate procedure - `instructions/setup-instance.md` - the first publish of a new instance +- `wikitool backup run` - the machine-data backup on its own + +## `backup init` + +Create the backup target's bare repository and make it `origin` when there is none. + +**Synopsis** + +- `wikitool backup init` + +**Options** + +```text + -h, --help Show this message and exit. +``` + +**Examples** + +```bash +tools/wikitool backup init +``` + +**Behaviour** + +- Reads the `backup` section of `.wikitool-local.json` and creates `/` and `/git`, the latter as a bare repository (`git init --bare -b main`). An existing bare repository there, or an empty directory, is used as it is. +- Sets `origin` to `/git` when the checkout has no `origin`. An `origin` that already pushes there is left as it is; one that pushes anywhere else stops the command. +- When `remotes.allowed_push_urls` exists and lacks the URL `origin` pushes to, prints the `remotes` section with that URL added, for the user to copy into `.wikitool-local.json`. It never writes that file: the field is a gate. +- Pushes nothing: the next `publish` sends the history to the new `origin`. + +**Properties** + +- Changes files: yes +- Idempotent: yes +- Atomic: Every precondition is checked first; a refusal changes nothing +- Counts toward the session call budget: yes +- Can reach the network: no + +**Exit codes** + +- `0` - success +- `1` - No `backup` section in `.wikitool-local.json`, or a malformed one - a target that is not absolute or lies inside the instance, a `machine_name` that cleans to nothing or to `git`, an `extra` path that resolves outside the instance or into `.git/` +- `1` - `/git` exists and is not a bare repository; nothing was changed +- `1` - `origin` already pushes to another URL; the message names both and nothing was changed + +**See also** + +- `wikitool backup run` - the machine-data half of the backup +- `wikitool publish` - pushes to the bare repository and runs the machine-data backup after it +- `docs/how-to/back-up-to-a-synced-folder.md` - the whole procedure, restoring included + +## `backup run` + +Mirror this machine's files that are not in git into `//`. + +**Synopsis** + +- `wikitool backup run` + +**Options** + +```text + -h, --help Show this message and exit. +``` + +**Examples** + +```bash +tools/wikitool backup run +``` + +**Behaviour** + +- Mirrors `.wikitool-local.json`, `ENVIRONMENT.md` if present, every file in `incoming/` but `.gitkeep`, and the `extra` paths of the `backup` section into `//`, then writes the manifest `.chemenu-backup.json` there: time, stack version, machine name, and per file its relative path, size and sha256. +- Never mirrors `mcp-upload/`, `reports/`, `.wikitool-tools.json`, `tools/.venv/`, the published skills or `.git/` - unless an `extra` path names one of the first ones. The secrets file lies outside the instance and is never read. +- Writes and deletes only under `//`. Deletes only a file the previous manifest lists and the instance no longer has; a file there the manifest does not list stays as it is. A file whose size and sha256 match the previous manifest is not rewritten. +- A symbolic link or junction inside a mirrored directory is skipped and named in the output, never followed. +- `publish` runs the same step after every push when `machine_data` is true. + +**Properties** + +- Changes files: yes +- Idempotent: yes +- Atomic: Per file - each is written to a temporary file and renamed, the manifest last; an interrupted run leaves whole files and the previous manifest +- Counts toward the session call budget: yes +- Can reach the network: no + +**Exit codes** + +- `0` - success +- `1` - No `backup` section in `.wikitool-local.json`, or a malformed one - a target that is not absolute or lies inside the instance, a `machine_name` that cleans to nothing or to `git`, an `extra` path that resolves outside the instance or into `.git/` +- `1` - `machine_data` is false +- `1` - `//` is not empty and has no manifest, or its manifest cannot be read; nothing was written +- `1` - The target is not a directory (the synced folder is missing or not mounted), or a write failed + +**See also** + +- `wikitool backup init` - sets up the target's bare repository +- `wikitool doctor` - the `backup` line reports the age of the last run +- `docs/how-to/back-up-to-a-synced-folder.md` - the whole procedure, restoring included diff --git a/docs/reference/commands/instance-health.md b/docs/reference/commands/instance-health.md index 8ff0c15..5702e6f 100644 --- a/docs/reference/commands/instance-health.md +++ b/docs/reference/commands/instance-health.md @@ -36,6 +36,8 @@ tools/wikitool doctor --json - KB conventions: `kb/CONVENTIONS.md` present, unsentinelled, and naming all three tool-owned section headings - a `FAIL` on any of the three. - Environment note: `ENVIRONMENT.md` is optional, so absent is `OK`; a still-templated one is a `WARN`. - Legacy settings (`legacy-settings`): one `FAIL` for each per-feature settings file of an earlier version (telemetry, remotes, upload, task tracker) that is still there. Nothing reads them any more, so the settings they hold are not in effect; the fix is `tools/wikitool config migrate`. +- Synced folder (`sync-folder`): a `WARN` when the git directory, links and junctions resolved, lies under a folder named by `OneDrive`, `OneDriveCommercial` or `OneDriveConsumer` (compared without regard to case on Windows) or under `~/Library/CloudStorage/` (macOS); `OK` otherwise, never a `FAIL`. +- Backup (`backup`): the `backup` section absent is `OK`, malformed is a `FAIL`. Configured, a `WARN` when `/git` is not a bare repository, `origin` does not push there, or - with `machine_data` true - no machine-data backup exists yet; otherwise `OK` with the age of the last machine-data backup. - MCP `submit` tool: whether the `upload` section of the settings is present, absent or malformed, its limits, and how many submissions wait in `mcp-upload/`. Absent is `OK` and means the write path does not exist at all; malformed is a `FAIL`. - Task tracker: the `tasks` section present, absent or malformed - absent is `OK` (no tracker configured), malformed is a `FAIL`. When `WIKITOOL_TASKS_CONFIG` is set, the profile it names is read instead of the `tasks`, `superproductivity` and `caldav` sections and the finding names it; a set variable that names no file is a `FAIL`, never `OK`. - For a configured `superproductivity` provider, the configured `access` path's own state: `access: "api"` reports whether its local REST API answers `GET /health` with a ready renderer right now, `access: "snapshot"` whether a backup file is ready. The other access path is never attempted, and neither state is ever a `FAIL`. @@ -67,5 +69,6 @@ tools/wikitool doctor --json - `instructions/preflight.md` - what `tool-paths` and `install-dir` point back to - `docs/reference/configuration.md` - the per-checkout configuration files and the tiered settings - `docs/explanation/telemetry-and-evaluation.md` - telemetry state and caps +- `docs/how-to/back-up-to-a-synced-folder.md` - what `sync-folder` and `backup` point to - `instructions/session-setup.md` - setting `WIKITOOL_SESSION_ID` diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 1ae8861..881584c 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -30,6 +30,7 @@ something you run. - [MCP submissions](#mcp-submissions) - [Site handlers](#site-handlers) - [Fetch routes](#fetch-routes) +- [Backup to a synced folder](#backup-to-a-synced-folder) - [Files of earlier versions](#files-of-earlier-versions) @@ -91,7 +92,8 @@ Each file is a JSON object of sections, one per feature, beside two keys of its `instance_id` sits in `wikitool.json` only. The first `wikitool config set` creates the file and the id; the id names the directory of the secrets file below. The sections are [telemetry](#telemetry), [remotes](#allowed-push-urls), [tasks, superproductivity and caldav](#task-tracker) and -[upload](#mcp-submissions), [sites](#site-handlers) and [fetch](#fetch-routes). `config set` names the sections it knows, and `doctor` reports a +[upload](#mcp-submissions), [sites](#site-handlers), [fetch](#fetch-routes) and +[backup](#backup-to-a-synced-folder). `config set` names the sections it knows, and `doctor` reports a section it does not read as a `WARN`. **One place per value.** A value in the file its tier does not name is an error naming the file, @@ -533,6 +535,55 @@ Example: ``` +## Backup to a synced folder + +The `backup` section of `.wikitool-local.json` backs this checkout up into a folder a sync client +uploads - OneDrive and the like - while the working tree itself stays outside it. The target gets +two kinds of content: + +``` +/git/ bare repository, the checkout's origin; publish pushes here +// mirror of this machine's files that are not in git, plus .chemenu-backup.json +``` + +`tools/wikitool backup init` creates `/git` and makes it `origin` when the checkout has none. +With `machine_data` on, every `publish` that pushed mirrors `.wikitool-local.json`, `ENVIRONMENT.md`, +what waits in `incoming/` and the `extra` paths into `//`; `tools/wikitool backup +run` does the same on its own. The secrets file is never among them - it lies outside the instance, +and a backup reads nothing outside it. Plain private values in `.wikitool-local.json`, an internal +host name for one, do reach the synced folder. + + +| Field | Tier | Type | Required | Default | Description | +|-------|------|------|----------|---------|-------------| +| `target` | checkout, gate | string | yes | - | The backup folder: an absolute path outside the instance. `backup init` creates `/git` in it, `backup run` writes only to `//`. | +| `machine_data` | checkout, gate | boolean | no | `false` | Mirror this machine's files after every `publish` that pushed, and with `backup run`: `.wikitool-local.json`, `ENVIRONMENT.md`, what waits in `incoming/`, and `extra`. | +| `machine_name` | checkout | string | no | the host name | The name of this machine's folder under the target. Every character outside letters, digits, `-` and `_` becomes `-`, and leading and trailing `-` are dropped; a name that is empty afterwards, or `git` in any case, is an error. | +| `extra` | checkout, gate | array of strings | no | `[]` | Further files or directories to mirror, relative to the instance. One that resolves outside the instance, or into `.git/`, is an error. | + +- **Absent:** No backup: `publish` behaves as without it, and `backup init` and `backup run` stop with exit 1 naming the section. +- **Malformed:** An error. `publish` stops with exit 1 before it commits anything, `backup init` and `backup run` stop with exit 1, and `doctor` reports `FAIL`. A broken section is never read as "no backup". + +Example: + +```json +{ + "backup": { + "target": "C:\\Users\\\\OneDrive - \\Chemenu-Backup", + "machine_data": true, + "machine_name": "WORK-LAPTOP", + "extra": [ + ".claude/settings.local.json" + ] + } +} +``` + + +`target`, `machine_data` and `extra` decide what leaves this machine, so they are marked `gate`: +`config set` exits 42 for them, and only you write them. The procedure, restoring included: +[back-up-to-a-synced-folder.md](../how-to/back-up-to-a-synced-folder.md). + ## Files of earlier versions Until version 10 every feature above had a file of its own at the root: `.wikitool-telemetry.json`, diff --git a/instructions/publish-cycle.md b/instructions/publish-cycle.md index b0eec12..be65cc8 100644 --- a/instructions/publish-cycle.md +++ b/instructions/publish-cycle.md @@ -52,7 +52,12 @@ consistent. to commit locally with `--no-push`, and run that only on their answer. Never push by hand (AGENTS.md invariant 5): the next `publish` that reaches the remote sends the local commit together with whatever is new. A local-only instance, which has no remote at all, passes - `--no-push` on every call ([setup-instance.md](setup-instance.md), step 4). + `--no-push` on every call ([setup-instance.md](setup-instance.md), step 3). +- **A `WARN` that the machine-data backup failed?** The commit and the push stand, and `publish` + exited 0. Show the user the line - it names the reason, a synced folder that is not there or not + writable, say. Do **not** run `publish` again: it would have nothing to push and back nothing up. + Once the user has fixed the cause, `tools/wikitool backup run` repeats the backup + ([docs/how-to/back-up-to-a-synced-folder.md](../docs/how-to/back-up-to-a-synced-folder.md)). - **Query or lint pass?** Neither auto-publishes. Run `publish` only if asked to. - **Nothing under `kb/` changed?** Skip steps 1 and 2; a change to `tools/` or `instructions/` does not affect the catalog. diff --git a/instructions/setup-instance.md b/instructions/setup-instance.md index ac8c52b..72a1324 100644 --- a/instructions/setup-instance.md +++ b/instructions/setup-instance.md @@ -122,6 +122,14 @@ one it needs before that. - Not given: stay local - then **every** later `tools/wikitool publish` needs a `--no-push` (which also drops its branch check, see step 1). Without it, `publish` ends with exit 1 before it commits anything, because there is no remote to publish to. + - No git server, but a folder a sync client uploads (OneDrive and the like): offer a backup + into it. The working tree stays where it is - outside that folder - and a bare repository in + the folder becomes `origin`, with the files of this machine that are not in git mirrored + beside it. The user writes the `backup` section of `.wikitool-local.json` by hand (its + fields are gates, so `config set` refuses them); then run `tools/wikitool backup init` and + show the user any `remotes.allowed_push_urls` entry it prints, for them to add. The whole + procedure is [docs/how-to/back-up-to-a-synced-folder.md](../docs/how-to/back-up-to-a-synced-folder.md). + Until it is done, treat the instance as local. 4. **Decision point - authoring conventions.** The release ships no filled-in conventions, only `kb/CONVENTIONS.md.template` and one `kb//COLLECTION.md.template` per collection. Both diff --git a/tools/CONTRACT.md b/tools/CONTRACT.md index e7d890b..e3062ce 100644 --- a/tools/CONTRACT.md +++ b/tools/CONTRACT.md @@ -136,6 +136,8 @@ upload accept write non-idempotent budget:counted exit:0,1, upload reject write non-idempotent budget:counted exit:0,1 Delete a submission's material, keeping only its ledger trail. sync write idempotent budget:counted exit:0,1,42 Fetch `/` and bring the local branch up to date with it. publish write non-idempotent budget:counted exit:0,1,42 Reconcile with `/`, then stage all changes, commit, and push. +backup init write idempotent budget:counted exit:0,1 Create the backup target's bare repository and make it `origin` when there is none. +backup run write idempotent budget:counted exit:0,1 Mirror this machine's files that are not in git into `//`. export guidelines write idempotent budget:counted exit:0,1,42 Render the guideline pages into one generated `GUIDELINES.md`, or push it into the captured repositories that opted in (**Guideline Push Gate**). work new write non-idempotent budget:counted exit:0,1 Scaffold `work//` for one workshop run. work close write non-idempotent budget:counted exit:0,1 Delete a finished workshop. @@ -2120,11 +2122,12 @@ Reconcile with `/`, then stage all changes, commit, and push. **EXIT STATUS** - 0 success +- 0 The machine-data backup after the push failed: a `WARN` line names the reason, and the commit and the push stand - 1 git failed - `git add`, `git commit`, `git push`, or the reconcile's automatic rebase (a conflict outside the generated files, an edited `kb/log.md`, or an uncommitted change on a path the incoming commits change) - 1 The push target (`--branch`) is not the checked-out branch, or HEAD is detached; the unborn branch of a fresh `git init` is not this case - 1 No `--no-push`, and the remote is not configured or cannot be reached; nothing was committed - 1 `--yes`/`-y` was passed - the flag does not exist and fails with an explicit error -- 1 The settings files do not pass their check, or `remotes.allowed_push_urls` is not a list of strings +- 1 The settings files do not pass their check, `remotes.allowed_push_urls` is not a list of strings, or the `backup` section is malformed; nothing was committed - 42 The remotes settings file of an earlier version is still there - 42 Mass-Update Gate: `--threshold` (default 10) or more counted files would be committed, or the `--confirm` token does not match this changeset - 42 Rebase-Review Gate: `/` moved and both sides changed the same file; the output lists the upstream commits, the overlapping files and their diff @@ -2132,11 +2135,12 @@ Reconcile with `/`, then stage all changes, commit, and push. **ON FAILURE** +- The machine-data backup after the push failed: a `WARN` line names the reason, and the commit and the push stand -> Show the `WARN` line to the user. Do not publish again - once the cause is fixed, `tools/wikitool backup run` repeats the backup - git failed - `git add`, `git commit`, `git push`, or the reconcile's automatic rebase (a conflict outside the generated files, an edited `kb/log.md`, or an uncommitted change on a path the incoming commits change) -> Do not retry and do not force - report and ask the user. `publish` has already made its one retry of a rejected push itself, where a reconcile resolved the rejection - The push target (`--branch`) is not the checked-out branch, or HEAD is detached; the unborn branch of a fresh `git init` is not this case -> Check out the branch you mean to publish, or pass `--branch `, then retry once - No `--no-push`, and the remote is not configured or cannot be reached; nothing was committed -> Show the message to the user and ask whether to commit locally with `--no-push`. Never push by hand - the next `publish` that reaches the remote sends that commit - `--yes`/`-y` was passed - the flag does not exist and fails with an explicit error -> Drop it. The Mass-Update Gate is cleared only with `--confirm ` from the gate's own refusal output -- The settings files do not pass their check, or `remotes.allowed_push_urls` is not a list of strings -> Show the error to the user and stop - a malformed section is not permission, and fixing it is theirs to do +- The settings files do not pass their check, `remotes.allowed_push_urls` is not a list of strings, or the `backup` section is malformed; nothing was committed -> Show the error to the user and stop - a malformed section is not permission, and fixing it is theirs to do - The remotes settings file of an earlier version is still there -> Show the output to the user verbatim and stop; `tools/wikitool config migrate` moves it, and the user decides when - Mass-Update Gate: `--threshold` (default 10) or more counted files would be committed, or the `--confirm` token does not match this changeset -> Show the user the command's full output verbatim and stop. Once they have approved it, run the re-run line the output prints, which carries `--confirm `. Without that token, or with a wrong, invented or superseded one, it exits 42 again with the current state - Rebase-Review Gate: `/` moved and both sides changed the same file; the output lists the upstream commits, the overlapping files and their diff -> Show the user the command's full output verbatim and stop. Once they have approved it, run the re-run line the output prints, which carries `--confirm-rebase `. Without that token, or with a wrong, invented or superseded one, it exits 42 again with the current state @@ -2165,6 +2169,7 @@ Reconcile with `/`, then stage all changes, commit, and push. - The remotes settings file of an earlier version is not read: while it exists this exits 42 and names `tools/wikitool config migrate`, so a restriction is never lost silently. - `--path` (repeatable) scopes the whole operation - gate count, staging and commit - to that subtree, with one exception: uncommitted generated files outside `--path` (`kb/index.md`, `kb/log.md`, `kb/provenance.md`, every `INDEX.md` under `kb/`) are staged and committed too, and a line names them - whether a reconcile or the change itself wrote them. They stay out of the gate count, and the `--confirm` token and the re-run line cover the `--path` given. Every other file outside `--path` is left as it is. - Commit message: `--message`, then a `Files changed:` paragraph listing every committed path. When `--message` ends in a paragraph git reads as a trailer block (`Co-Authored-By:` and the like), that block stays the last paragraph and the list goes in front of it, since git reads trailers from the last paragraph only. `git interpret-trailers` decides whether there is such a block, and the list moves only when git reads the same trailers from the result; otherwise it is appended at the end. `--message` is not part of any gate token. +- Machine-data backup: with a `backup` section in `.wikitool-local.json` whose `machine_data` is true, a call that pushed then mirrors this machine's files that are not in git into `//` - the same step as `backup run`. A failure there prints a `WARN` line with the reason and still exits 0, since the commit and the push stand. No backup with `--no-push`, after a failed push, or when there was nothing to push. A malformed `backup` section stops the call with exit 1 before anything is committed, `--no-push` included. - After a successful commit or push whose changed files include `tools/`, `types/`, `instructions/`, `AGENTS.md` or a path ending in `CONTRACT.md`, prints one reminder line: the phase past this point (an issue-body rewrite, `docs/` staleness, a changelog entry's accuracy) is not covered by `docs verify`, `instructions verify` or `pytest`. It is not a gate: no exit code change, nothing to clear, and silent for an ordinary content publish. **SEE ALSO** @@ -2172,6 +2177,110 @@ Reconcile with `/`, then stage all changes, commit, and push. - `wikitool sync` - the same reconcile on its own, without committing or pushing - `instructions/gates.md` - the gate procedure - `instructions/setup-instance.md` - the first publish of a new instance +- `wikitool backup run` - the machine-data backup on its own + +#### `backup init` + +Create the backup target's bare repository and make it `origin` when there is none. + +**SYNOPSIS** + +- `wikitool backup init` + +**PROPERTIES** + +- effect: write +- idempotent: yes +- atomic: Every precondition is checked first; a refusal changes nothing +- budget: counted +- network: no + +**EXAMPLES** + +- `tools/wikitool backup init` + +**EXIT STATUS** + +- 0 success +- 1 No `backup` section in `.wikitool-local.json`, or a malformed one - a target that is not absolute or lies inside the instance, a `machine_name` that cleans to nothing or to `git`, an `extra` path that resolves outside the instance or into `.git/` +- 1 `/git` exists and is not a bare repository; nothing was changed +- 1 `origin` already pushes to another URL; the message names both and nothing was changed + +**ON FAILURE** + +- No `backup` section in `.wikitool-local.json`, or a malformed one - a target that is not absolute or lies inside the instance, a `machine_name` that cleans to nothing or to `git`, an `extra` path that resolves outside the instance or into `.git/` -> Show the message to the user and stop. The section is theirs to write or fix by hand - `config set` refuses its fields; docs/how-to/back-up-to-a-synced-folder.md has the procedure +- `/git` exists and is not a bare repository; nothing was changed -> Show the message to the user - whether that folder moves, or the target does, is their decision +- `origin` already pushes to another URL; the message names both and nothing was changed -> Show both URLs to the user and stop. Never repoint `origin` yourself - `git remote set-url` is theirs to run + +**NEVER** + +- Never add the printed URL to `remotes.allowed_push_urls` yourself - that field is a gate, and editing it is the user's. + +**NOTES** + +- Reads the `backup` section of `.wikitool-local.json` and creates `/` and `/git`, the latter as a bare repository (`git init --bare -b main`). An existing bare repository there, or an empty directory, is used as it is. +- Sets `origin` to `/git` when the checkout has no `origin`. An `origin` that already pushes there is left as it is; one that pushes anywhere else stops the command. +- When `remotes.allowed_push_urls` exists and lacks the URL `origin` pushes to, prints the `remotes` section with that URL added, for the user to copy into `.wikitool-local.json`. It never writes that file: the field is a gate. +- Pushes nothing: the next `publish` sends the history to the new `origin`. + +**SEE ALSO** + +- `wikitool backup run` - the machine-data half of the backup +- `wikitool publish` - pushes to the bare repository and runs the machine-data backup after it +- `docs/how-to/back-up-to-a-synced-folder.md` - the whole procedure, restoring included + +#### `backup run` + +Mirror this machine's files that are not in git into `//`. + +**SYNOPSIS** + +- `wikitool backup run` + +**PROPERTIES** + +- effect: write +- idempotent: yes +- atomic: Per file - each is written to a temporary file and renamed, the manifest last; an interrupted run leaves whole files and the previous manifest +- budget: counted +- network: no + +**EXAMPLES** + +- `tools/wikitool backup run` + +**EXIT STATUS** + +- 0 success +- 1 No `backup` section in `.wikitool-local.json`, or a malformed one - a target that is not absolute or lies inside the instance, a `machine_name` that cleans to nothing or to `git`, an `extra` path that resolves outside the instance or into `.git/` +- 1 `machine_data` is false +- 1 `//` is not empty and has no manifest, or its manifest cannot be read; nothing was written +- 1 The target is not a directory (the synced folder is missing or not mounted), or a write failed + +**ON FAILURE** + +- No `backup` section in `.wikitool-local.json`, or a malformed one - a target that is not absolute or lies inside the instance, a `machine_name` that cleans to nothing or to `git`, an `extra` path that resolves outside the instance or into `.git/` -> Show the message to the user and stop. The section is theirs to write or fix by hand - `config set` refuses its fields; docs/how-to/back-up-to-a-synced-folder.md has the procedure +- `machine_data` is false -> Show the message to the user; turning it on is their edit of `.wikitool-local.json` +- `//` is not empty and has no manifest, or its manifest cannot be read; nothing was written -> Show the message to the user - the folder is not one this backup wrote. They set another `machine_name` or empty it +- The target is not a directory (the synced folder is missing or not mounted), or a write failed -> Show the message to the user; once the cause is fixed, run `backup run` again + +**NEVER** + +- Never point `extra` at a path outside the instance to back something else up - the command refuses it, and the secrets belong in no synced folder. + +**NOTES** + +- Mirrors `.wikitool-local.json`, `ENVIRONMENT.md` if present, every file in `incoming/` but `.gitkeep`, and the `extra` paths of the `backup` section into `//`, then writes the manifest `.chemenu-backup.json` there: time, stack version, machine name, and per file its relative path, size and sha256. +- Never mirrors `mcp-upload/`, `reports/`, `.wikitool-tools.json`, `tools/.venv/`, the published skills or `.git/` - unless an `extra` path names one of the first ones. The secrets file lies outside the instance and is never read. +- Writes and deletes only under `//`. Deletes only a file the previous manifest lists and the instance no longer has; a file there the manifest does not list stays as it is. A file whose size and sha256 match the previous manifest is not rewritten. +- A symbolic link or junction inside a mirrored directory is skipped and named in the output, never followed. +- `publish` runs the same step after every push when `machine_data` is true. + +**SEE ALSO** + +- `wikitool backup init` - sets up the target's bare repository +- `wikitool doctor` - the `backup` line reports the age of the last run +- `docs/how-to/back-up-to-a-synced-folder.md` - the whole procedure, restoring included ### Guideline export @@ -3907,6 +4016,8 @@ Check that this instance is correctly configured. - KB conventions: `kb/CONVENTIONS.md` present, unsentinelled, and naming all three tool-owned section headings - a `FAIL` on any of the three. - Environment note: `ENVIRONMENT.md` is optional, so absent is `OK`; a still-templated one is a `WARN`. - Legacy settings (`legacy-settings`): one `FAIL` for each per-feature settings file of an earlier version (telemetry, remotes, upload, task tracker) that is still there. Nothing reads them any more, so the settings they hold are not in effect; the fix is `tools/wikitool config migrate`. +- Synced folder (`sync-folder`): a `WARN` when the git directory, links and junctions resolved, lies under a folder named by `OneDrive`, `OneDriveCommercial` or `OneDriveConsumer` (compared without regard to case on Windows) or under `~/Library/CloudStorage/` (macOS); `OK` otherwise, never a `FAIL`. +- Backup (`backup`): the `backup` section absent is `OK`, malformed is a `FAIL`. Configured, a `WARN` when `/git` is not a bare repository, `origin` does not push there, or - with `machine_data` true - no machine-data backup exists yet; otherwise `OK` with the age of the last machine-data backup. - MCP `submit` tool: whether the `upload` section of the settings is present, absent or malformed, its limits, and how many submissions wait in `mcp-upload/`. Absent is `OK` and means the write path does not exist at all; malformed is a `FAIL`. - Task tracker: the `tasks` section present, absent or malformed - absent is `OK` (no tracker configured), malformed is a `FAIL`. When `WIKITOOL_TASKS_CONFIG` is set, the profile it names is read instead of the `tasks`, `superproductivity` and `caldav` sections and the finding names it; a set variable that names no file is a `FAIL`, never `OK`. - For a configured `superproductivity` provider, the configured `access` path's own state: `access: "api"` reports whether its local REST API answers `GET /health` with a ready renderer right now, `access: "snapshot"` whether a backup file is ready. The other access path is never attempted, and neither state is ever a `FAIL`. @@ -3925,6 +4036,7 @@ Check that this instance is correctly configured. - `instructions/preflight.md` - what `tool-paths` and `install-dir` point back to - `docs/reference/configuration.md` - the per-checkout configuration files and the tiered settings - `docs/explanation/telemetry-and-evaluation.md` - telemetry state and caps +- `docs/how-to/back-up-to-a-synced-folder.md` - what `sync-folder` and `backup` point to - `instructions/session-setup.md` - setting `WIKITOOL_SESSION_ID` ## Design notes diff --git a/tools/README.md b/tools/README.md index c853f98..fb7ea16 100644 --- a/tools/README.md +++ b/tools/README.md @@ -130,6 +130,7 @@ tools/ browser.py the optional browser component (Playwright, `requirements-browser.txt`): `raw login`'s visible window and the headless fetch of a page a handler declares `render` - imported lazily, so a venv without it runs everything else sites/ the site handlers behind `raw fetch`: a registry (`__init__.py`), the JSON file format (`site.schema.json`, `data.py`) and the stack's handlers as one file per website under `handlers/`, each stating the standard form of its URLs, the page furniture to leave out of the derived text, hint lines, which links are subpages for `raw fetch --scope`, a website's login page and the sign of a signed-in page, and whether its pages need a browser - all of it data (rewrites, selectors, text); the instance's own sit in `sites/` at the root filters/ the import filters behind `raw derive`: a registry (`__init__.py`) and one module per format family (HTML, mail, Outlook MSG, Office, PDF, ZIP), each registering the extensions it reads - standard library, or a package imported lazily so a missing one fails only its own format + backup.py `backup init`/`backup run`'s core and the step `publish` runs after a push: the bare repository under `/git`, the machine-data mirror under `//` with its manifest, and which synced folders `doctor` knows repo_capture.py `raw capture`/`raw status`'s core: resolve a ref rule, fetch it into a bare cache, select files by glob as blobs, the `_capture.json` manifest - git with the host's credentials, never a prompt; plus the one write back, a single-file commit pushed without force for `export guidelines` guideline_export.py `export guidelines`' core: select pages by `search` predicates, render them into one deterministic GUIDELINES.md, and recognise an export again (`EXPORT_MARKER`) - with no CLI attached search/ pluggable search backends, plus service.py - the search core diff --git a/tools/chemenu/backup.py b/tools/chemenu/backup.py new file mode 100644 index 0000000..7aba4f4 --- /dev/null +++ b/tools/chemenu/backup.py @@ -0,0 +1,563 @@ +"""Backing an instance up into a folder a sync client uploads (OneDrive and the like). + +The working tree stays out of that folder: a sync client locks files under +`.git` for a moment (which breaks `git gc` and `repack`), offloads them as +placeholders, and makes conflict copies inside `.git` as soon as a second device +syncs the same folder. What goes into the synced folder is a backup target with +this layout (`config_shapes.BACKUP` is the section that names it): + + /git/ a bare repository, the checkout's `origin`; `publish` pushes to it + // a mirror of this machine's files that are not in git, + plus the manifest `.chemenu-backup.json` + +Git writes into the bare repository only on a push, with one writer, so the +sync client sees mostly new, immutable object files. + +What a machine-data run guarantees - each one has its test: + +- It writes and deletes only under `//`. Nothing outside it + changes - not `/git`, not another machine's folder. +- It deletes only what the previous manifest lists and is gone locally. A file + in the folder the manifest does not list stays as it is. +- It refuses a non-empty `//` without a manifest, before it + writes anything: it never mirrors into a folder it did not write. +- It reads nothing outside the instance. An `extra` path that resolves outside + is an error of the section; a symbolic link (or junction) inside a mirrored + directory is skipped and reported, never followed; `.git` is never mirrored. +- Every file is written atomically - a temporary file in the same directory, + then `os.replace` - and the manifest last, so an interrupted run leaves whole + files and the previous run's manifest. + +The secrets file lives outside the instance (`config.secrets_file`), so the last +two rules keep it out of the synced folder as well. +""" +from __future__ import annotations + +import hashlib +import json +import os +import socket +import subprocess +import tempfile +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +from chemenu import config, config_shapes, settings, toolpaths +from chemenu.errors import ValidationError + +MANIFEST_NAME = ".chemenu-backup.json" +MANIFEST_SCHEMA = 1 +INCOMING_DIR = "incoming" +INCOMING_KEEP = ".gitkeep" +GIT_NAME = ".git" +# The files at the root of the instance a machine-data run mirrors when they exist. +ROOT_FILES = (config.LOCAL_SETTINGS_FILENAME, config.ENVIRONMENT_FILE) + +# Where a sync client keeps its folder: the variables the OneDrive client sets on +# Windows, and the folder macOS's File Provider puts every sync client under. +SYNC_ENV_VARS = ("OneDrive", "OneDriveCommercial", "OneDriveConsumer") +MAC_CLOUD_STORAGE = ("Library", "CloudStorage") +# Windows compares paths without regard to case; a test sets this to compare +# that way on another platform. +IGNORE_CASE = os.name == "nt" + +# OneDrive holds a file it is uploading open for a moment; `os.replace` onto it +# then fails with a sharing violation that is gone a second later. +_REPLACE_ATTEMPTS = 5 +_REPLACE_PAUSE_SECONDS = 0.4 + + +class BackupError(ValidationError): + """A backup step refused or failed; exit 1 with the message.""" + + +@dataclass(frozen=True) +class BackupConfig: + target: Path + machine_data: bool + machine_name: str + extra: tuple[str, ...] + + @property + def git_dir(self) -> Path: + return self.target / config_shapes.BACKUP_GIT_DIR + + @property + def machine_dir(self) -> Path: + return self.target / self.machine_name + + @property + def manifest(self) -> Path: + return self.machine_dir / MANIFEST_NAME + + +# --------------------------------------------------------------------------- +# Paths +# --------------------------------------------------------------------------- + + +def _is_link(path: Path) -> bool: + is_junction = getattr(path, "is_junction", None) # Python 3.12+ + return path.is_symlink() or bool(is_junction and is_junction()) + + +def _real(path: Path) -> Path: + return Path(os.path.realpath(path)) + + +def is_under(path: Path, root: Path, ignore_case: Optional[bool] = None) -> bool: + """Whether `path` is `root` or lies below it, both taken as given.""" + ignore_case = IGNORE_CASE if ignore_case is None else ignore_case + inner, outer = str(path), str(root).rstrip("/\\") + if ignore_case: + inner, outer = inner.casefold(), outer.casefold() + return inner == outer or inner.startswith(outer + os.sep) or inner.startswith(outer + "/") + + +def _names_git(parts) -> bool: + return any(part.casefold() == GIT_NAME for part in parts) + + +def _safe_relative(rel: str) -> bool: + """Whether `rel`, a `/`-separated path, stays below the folder it is joined + to on every platform: no empty, `.` or `..` part, no backslash, no drive.""" + parts = rel.split("/") + return bool(rel) and all(part not in ("", ".", "..") and "\\" not in part and ":" not in part + for part in parts) + + +def _within(machine_dir: Path, rel: str) -> Path: + """`rel` below `machine_dir`, refusing a path that would leave it - by its + spelling, or through a link somewhere in the folder.""" + if not _safe_relative(rel): + raise BackupError(f"{rel!r} is not a path below {machine_dir} - nothing further was written") + dest = machine_dir.joinpath(*rel.split("/")) + current = machine_dir + for part in rel.split("/")[:-1]: + current = current / part + if _is_link(current): + raise BackupError(f"{current} is a link - a backup writes nothing through one") + return dest + + +# --------------------------------------------------------------------------- +# Reading the section +# --------------------------------------------------------------------------- + + +def _extra_problem(root: Path, real_root: Path, entry: str) -> Optional[str]: + if _names_git(Path(entry).parts): + return f"backup.extra holds {entry!r}, which names {GIT_NAME}/ - it is never backed up" + try: + resolved = (root / entry).resolve() + except (OSError, RuntimeError) as exc: + return f"backup.extra holds {entry!r}, which cannot be resolved ({exc})" + if not is_under(resolved, real_root): + return (f"backup.extra holds {entry!r}, which resolves to {resolved}, outside the " + "instance - a backup reads nothing outside it") + if _names_git(resolved.relative_to(real_root).parts): + return f"backup.extra holds {entry!r}, which resolves into {GIT_NAME}/ - it is never backed up" + return None + + +def read_config(root: "Path | str | None" = None) -> Optional[BackupConfig]: + """The `backup` section, or None when the settings carry none. Raises + `ValidationError` for a section that is broken - including what the record + cannot judge without the instance: a target inside it, an `extra` path that + leaves it, a host name that cleans to nothing.""" + root = Path(config.ROOT if root is None else root) + key = config_shapes.BACKUP.key + section = settings.load(root, only=(key,)).section(key) + if not section.configured: + return None + problems = [] + real_root = _real(root) + + target = Path(section["target"]) + if not os.path.isabs(section["target"]): + problems.append(f"backup.target {section['target']!r} is not an absolute path on this machine") + else: + real_target = _real(target) + if is_under(real_target, real_root): + problems.append(f"backup.target {target} lies inside the instance - it belongs outside it") + + if "machine_name" in section.given: + name, origin = section["machine_name"], "" + else: + name, origin = socket.gethostname(), " (the host name, since machine_name is not set)" + problem = config_shapes.machine_name_problem(name) + if problem: + problems.append(f"backup.machine_name {name!r}{origin} {problem} - set machine_name") + machine_name = config_shapes.machine_name(name) + + if not problems: + for inner in (target / config_shapes.BACKUP_GIT_DIR, target / machine_name): + if is_under(real_root, _real(inner)): + problems.append(f"the instance lies inside {inner}, which the backup writes to") + + for entry in section["extra"]: + found = _extra_problem(root, real_root, entry) + if found: + problems.append(found) + + if problems: + raise ValidationError("; ".join(problems)) + return BackupConfig(target=target, machine_data=section["machine_data"], + machine_name=machine_name, extra=tuple(section["extra"])) + + +def require_config(root: "Path | str | None" = None) -> BackupConfig: + found = read_config(root) + if found is None: + raise BackupError( + f"no backup section in {config.LOCAL_SETTINGS_FILENAME} - the user adds it by hand " + "(docs/how-to/back-up-to-a-synced-folder.md)" + ) + return found + + +# --------------------------------------------------------------------------- +# git +# --------------------------------------------------------------------------- + + +def _git(args: list[str], cwd: Path) -> subprocess.CompletedProcess: + return subprocess.run([toolpaths.git(), *args], cwd=cwd, capture_output=True, text=True, + encoding="utf-8", errors="replace") + + +def is_bare_repo(path: Path) -> bool: + if not path.is_dir(): + return False + result = _git(["--git-dir", str(path), "rev-parse", "--is-bare-repository"], cwd=path) + return result.returncode == 0 and result.stdout.strip() == "true" + + +def push_url(root: Path, remote: str = "origin") -> Optional[str]: + result = _git(["remote", "get-url", "--push", remote], cwd=root) + if result.returncode != 0: + return None + return result.stdout.strip() or None + + +def same_location(url: str, path: Path, root: Path) -> bool: + """Whether the remote URL `url` names the directory `path`. A URL with a + scheme other than `file://`, or an scp-like `host:path`, never does.""" + local = url[len("file://"):] if url.startswith("file://") else url + if "://" in local or (":" in local and not os.path.isabs(local)): + return False + a, b = os.path.normcase(_real(root / local)), os.path.normcase(_real(path)) + return a.casefold() == b.casefold() if IGNORE_CASE else a == b + + +def _directory_state(path: Path) -> str: + if not path.exists() and not _is_link(path): + return "absent" + if path.is_dir() and not _is_link(path) and not any(path.iterdir()): + return "empty" + return "bare" if is_bare_repo(path) else "other" + + +@dataclass +class InitResult: + git_dir: Path + created_repo: bool + set_origin: bool + allowlist_entry: Optional[str] + + +def allowlist_entry(root: Path, url: str) -> Optional[str]: + """The `remotes` section as it would read with `url` added, when it exists + and lacks `url` - printed for the user to copy, never written: the field is + a gate.""" + key = config_shapes.REMOTES.key + section = settings.load(root, only=(key,)).section(key) + if not section.configured: + return None + allowed = list(section["allowed_push_urls"]) + if url in allowed: + return None + return json.dumps({key: {"allowed_push_urls": [*allowed, url]}}, indent=2, ensure_ascii=False) + + +def init(cfg: BackupConfig, root: "Path | str | None" = None) -> InitResult: + """Create `/git` as a bare repository and make it `origin` when + there is none. Checks everything first and changes nothing on a refusal.""" + root = Path(config.ROOT if root is None else root) + state = _directory_state(cfg.git_dir) + if state == "other": + raise BackupError( + f"{cfg.git_dir} exists and is not a bare git repository - nothing was changed. Move it " + "away, or point backup.target at another folder" + ) + current = push_url(root) + if current is not None and not same_location(current, cfg.git_dir, root): + raise BackupError( + f"origin already pushes to {current}, not to {cfg.git_dir} - nothing was changed. " + f"Whether origin moves is the user's decision: git remote set-url origin \"{cfg.git_dir}\"" + ) + + created = False + if state != "bare": + cfg.git_dir.mkdir(parents=True, exist_ok=True) + result = _git(["init", "--bare", "--quiet", "-b", "main", str(cfg.git_dir)], cwd=root) + if result.returncode != 0: + raise BackupError(f"git init --bare {cfg.git_dir} failed:\n{result.stderr}") + created = True + + set_origin = False + if current is None: + result = _git(["remote", "add", "origin", str(cfg.git_dir)], cwd=root) + if result.returncode != 0: + raise BackupError(f"git remote add origin failed:\n{result.stderr}") + set_origin, current = True, str(cfg.git_dir) + return InitResult(cfg.git_dir, created, set_origin, allowlist_entry(root, current)) + + +# --------------------------------------------------------------------------- +# Machine data +# --------------------------------------------------------------------------- + + +@dataclass +class Collected: + files: dict = field(default_factory=dict) # relative posix path -> source Path + skipped: list = field(default_factory=list) # " ()" + + +def _walk(path: Path, rel: str, out: Collected) -> None: + if _is_link(path): + out.skipped.append(f"{rel} (a link - not followed)") + return + if path.name.casefold() == GIT_NAME: + return + if path.is_file(): + if rel == MANIFEST_NAME: + out.skipped.append(f"{rel} (the manifest's own name)") + elif not _safe_relative(rel): + out.skipped.append(f"{rel} (a name that cannot be mirrored on every platform)") + else: + out.files.setdefault(rel, path) + return + if path.is_dir(): + for child in sorted(path.iterdir()): + _walk(child, f"{rel}/{child.name}" if rel else child.name, out) + + +def collect(cfg: BackupConfig, root: "Path | str | None" = None) -> Collected: + """What a machine-data run mirrors: the root files that exist, everything in + `incoming/` but its `.gitkeep`, and `extra`.""" + root = Path(config.ROOT if root is None else root) + real_root = _real(root) + out = Collected() + for name in ROOT_FILES: + if (root / name).exists() or _is_link(root / name): + _walk(root / name, name, out) + incoming = root / INCOMING_DIR + if _is_link(incoming): + out.skipped.append(f"{INCOMING_DIR} (a link - not followed)") + elif incoming.is_dir(): + for child in sorted(incoming.iterdir()): + if child.name != INCOMING_KEEP: + _walk(child, f"{INCOMING_DIR}/{child.name}", out) + for entry in cfg.extra: + resolved = (root / entry).resolve() + if not is_under(resolved, real_root): # read_config refuses this; never read it anyway + raise BackupError(f"backup.extra {entry!r} resolves outside the instance") + if not resolved.exists(): + out.skipped.append(f"{entry} (not present)") + continue + rel = resolved.relative_to(real_root).as_posix() + _walk(resolved, "" if rel == "." else rel, out) + return out + + +def _digest(path: Path) -> tuple[int, str]: + sha, size = hashlib.sha256(), 0 + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + sha.update(chunk) + size += len(chunk) + return size, sha.hexdigest() + + +def _replace(tmp: str, dest: Path) -> None: + for attempt in range(_REPLACE_ATTEMPTS): + try: + os.replace(tmp, dest) + return + except PermissionError: + if attempt == _REPLACE_ATTEMPTS - 1: + raise + time.sleep(_REPLACE_PAUSE_SECONDS) + + +def _write_atomic(dest: Path, chunks) -> tuple[int, str]: + dest.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(prefix=".chemenu-", suffix=".tmp", dir=str(dest.parent)) + sha, size = hashlib.sha256(), 0 + try: + with os.fdopen(fd, "wb") as out: + for chunk in chunks: + out.write(chunk) + sha.update(chunk) + size += len(chunk) + out.flush() + os.fsync(out.fileno()) + _replace(tmp, dest) + except BaseException: + Path(tmp).unlink(missing_ok=True) + raise + return size, sha.hexdigest() + + +def _copy_atomic(src: Path, dest: Path) -> tuple[int, str]: + with open(src, "rb") as handle: + return _write_atomic(dest, iter(lambda: handle.read(1 << 20), b"")) + + +def read_manifest(machine_dir: Path) -> Optional[dict]: + """The manifest of the last run into `machine_dir`, or None when there is + none. Raises `BackupError` for one that cannot be read as one.""" + path = machine_dir / MANIFEST_NAME + if not path.exists(): + return None + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise BackupError(f"{path} cannot be read ({exc}) - nothing was written") from None + files = data.get("files") if isinstance(data, dict) else None + if not isinstance(files, list) or not all( + isinstance(entry, dict) and isinstance(entry.get("path"), str) + and _safe_relative(entry["path"]) and isinstance(entry.get("size"), int) + and isinstance(entry.get("sha256"), str) + for entry in files + ): + raise BackupError(f"{path} is not a backup manifest - nothing was written") + return data + + +@dataclass +class RunResult: + machine_dir: Path + written: list = field(default_factory=list) + unchanged: list = field(default_factory=list) + removed: list = field(default_factory=list) + skipped: list = field(default_factory=list) + + def summary(self) -> str: + line = (f"Machine data backed up to {self.machine_dir}: {len(self.written)} written, " + f"{len(self.unchanged)} unchanged, {len(self.removed)} removed") + if self.skipped: + line += f"; skipped: {', '.join(self.skipped)}" + return line + + +def _prune_empty(directory: Path, stop: Path) -> None: + while directory != stop and is_under(directory, stop): + try: + directory.rmdir() + except OSError: + return + directory = directory.parent + + +def _stack_version() -> Optional[str]: + from chemenu import version as version_mod + + try: + return str(version_mod.read_version()) + except version_mod.VersionError: + return None + + +def run(cfg: BackupConfig, root: "Path | str | None" = None) -> RunResult: + """Mirror this machine's files into `//` and write its + manifest. See the module docstring for what it guarantees.""" + root = Path(config.ROOT if root is None else root) + if not cfg.target.is_dir(): + raise BackupError( + f"the backup target {cfg.target} is not a directory - nothing was written. Is the " + "synced folder there? `tools/wikitool backup init` creates the target" + ) + machine_dir = cfg.machine_dir + if _is_link(machine_dir) or (machine_dir.exists() and not machine_dir.is_dir()): + raise BackupError(f"{machine_dir} is not a plain directory - nothing was written") + previous = read_manifest(machine_dir) + if previous is None and machine_dir.is_dir() and any(machine_dir.iterdir()): + raise BackupError( + f"{machine_dir} is not empty and has no {MANIFEST_NAME} - it is not a folder this " + "backup wrote, so nothing was written. Set another backup.machine_name, or empty it" + ) + + collected = collect(cfg, root) + result = RunResult(machine_dir, skipped=list(collected.skipped)) + before = {entry["path"]: entry for entry in (previous or {}).get("files", [])} + entries = [] + machine_dir.mkdir(exist_ok=True) + for rel, source in sorted(collected.files.items()): + dest = _within(machine_dir, rel) + size, digest = _digest(source) + old = before.get(rel) + if (old and old["size"] == size and old["sha256"] == digest and dest.is_file() + and not _is_link(dest) and dest.stat().st_size == size): + result.unchanged.append(rel) + else: + size, digest = _copy_atomic(source, dest) + result.written.append(rel) + entries.append({"path": rel, "size": size, "sha256": digest}) + + for rel in sorted(set(before) - set(collected.files)): + dest = _within(machine_dir, rel) + if dest.is_file() and not _is_link(dest): + dest.unlink() + result.removed.append(rel) + _prune_empty(dest.parent, machine_dir) + + manifest = { + "schema": MANIFEST_SCHEMA, + "created": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "stack_version": _stack_version(), + "machine": cfg.machine_name, + "files": entries, + } + text = json.dumps(manifest, indent=2, ensure_ascii=False) + "\n" + _write_atomic(cfg.manifest, [text.encode("utf-8")]) + return result + + +# --------------------------------------------------------------------------- +# Synced folders (`doctor`) +# --------------------------------------------------------------------------- + + +def sync_roots() -> list[Path]: + """The folders a sync client keeps on this machine, as far as it says.""" + roots = [Path(value) for value in (os.environ.get(var, "").strip() for var in SYNC_ENV_VARS) if value] + roots.append(Path.home().joinpath(*MAC_CLOUD_STORAGE)) + return roots + + +def synced_folder_of(path: Path) -> Optional[Path]: + """The synced folder `path` lies in, links and junctions resolved on both + sides, or None.""" + real = _real(path) + for candidate in sync_roots(): + if is_under(real, _real(candidate)): + return candidate + return None + + +def age_text(created: str, now: Optional[datetime] = None) -> str: + try: + then = datetime.fromisoformat(created) + except (TypeError, ValueError): + return "of unknown age" + seconds = ((now or datetime.now(timezone.utc)) - then).total_seconds() + for limit, unit in ((86400, "d"), (3600, "h"), (60, "min")): + if seconds >= limit: + return f"{int(seconds // limit)} {unit} old" + return "less than a minute old" diff --git a/tools/chemenu/cli.py b/tools/chemenu/cli.py index 6b0f12a..a2b76e6 100644 --- a/tools/chemenu/cli.py +++ b/tools/chemenu/cli.py @@ -26,6 +26,7 @@ from chemenu import toolpaths # noqa: E402 try: from chemenu.commands import ( _util, + backup_cmd, cite_cmd, config_cmd, dist_cmd, @@ -182,6 +183,7 @@ app.add_typer(migrate_cmd.app, name="migrate") app.add_typer(task_cmd.app, name="task") app.add_typer(export_cmd.app, name="export") app.add_typer(config_cmd.app, name="config") +app.add_typer(backup_cmd.app, name="backup") app.command("new")(new_page.new_page_command) app.command("touch")(touch_module.touch_command) app.command("rename")(page_ops.rename_command) diff --git a/tools/chemenu/cli_contract.py b/tools/chemenu/cli_contract.py index a8503bc..31540f2 100644 --- a/tools/chemenu/cli_contract.py +++ b/tools/chemenu/cli_contract.py @@ -291,7 +291,7 @@ GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = ( "upload list", "upload show", "upload accept", "upload reject", )), ("Git", ( - "sync", "publish", + "sync", "publish", "backup init", "backup run", )), ("Guideline export", ( "export guidelines", diff --git a/tools/chemenu/commands/backup_cmd.py b/tools/chemenu/commands/backup_cmd.py new file mode 100644 index 0000000..dfee2db --- /dev/null +++ b/tools/chemenu/commands/backup_cmd.py @@ -0,0 +1,169 @@ +"""`wikitool backup init` and `backup run` - back an instance up into a synced folder. + +The mechanism and what a run guarantees are `chemenu.backup`'s; this module only +turns its results and refusals into output and exit codes. `publish` runs the +same machine-data step after a push (`git_publish.publish_command`). +""" +from __future__ import annotations + +import typer + +from chemenu import backup, cli_contract, config +from chemenu.commands._util import fail, success +from chemenu.errors import ChemenuError + +app = typer.Typer(help="Back this instance up into a synced folder (OneDrive and the like).") + +HOW_TO = "docs/how-to/back-up-to-a-synced-folder.md" + +_SECTION_FAILURE = cli_contract.Failure( + cause=f"No `backup` section in `{config.LOCAL_SETTINGS_FILENAME}`, or a malformed one - a " + "target that is not absolute or lies inside the instance, a `machine_name` that cleans to " + "nothing or to `git`, an `extra` path that resolves outside the instance or into `.git/`", + reaction=f"Show the message to the user and stop. The section is theirs to write or fix by " + f"hand - `config set` refuses its fields; {HOW_TO} has the procedure", +) + + +@app.command("init") +@cli_contract.record(cli_contract.CommandRecord( + path="backup init", + summary="Create the backup target's bare repository and make it `origin` when there is none.", + synopsis=(cli_contract.Variant(usage="backup init"),), + properties=cli_contract.Properties( + effect=cli_contract.Effect.WRITE, + idempotent=cli_contract.Idempotent.YES, + atomic="Every precondition is checked first; a refusal changes nothing", + budget=cli_contract.Budget.COUNTED, + ), + notes=( + "Reads the `backup` section of `.wikitool-local.json` and creates `/` and " + "`/git`, the latter as a bare repository (`git init --bare -b main`). An existing " + "bare repository there, or an empty directory, is used as it is.", + "Sets `origin` to `/git` when the checkout has no `origin`. An `origin` that " + "already pushes there is left as it is; one that pushes anywhere else stops the command.", + "When `remotes.allowed_push_urls` exists and lacks the URL `origin` pushes to, prints the " + "`remotes` section with that URL added, for the user to copy into " + "`.wikitool-local.json`. It never writes that file: the field is a gate.", + "Pushes nothing: the next `publish` sends the history to the new `origin`.", + ), + failures=( + _SECTION_FAILURE, + cli_contract.Failure( + cause="`/git` exists and is not a bare repository; nothing was changed", + reaction="Show the message to the user - whether that folder moves, or the target " + "does, is their decision", + ), + cli_contract.Failure( + cause="`origin` already pushes to another URL; the message names both and nothing was " + "changed", + reaction="Show both URLs to the user and stop. Never repoint `origin` yourself - " + "`git remote set-url` is theirs to run", + ), + ), + examples=("tools/wikitool backup init",), + never=( + "Never add the printed URL to `remotes.allowed_push_urls` yourself - that field is a gate, " + "and editing it is the user's.", + ), + see_also=( + "`wikitool backup run` - the machine-data half of the backup", + "`wikitool publish` - pushes to the bare repository and runs the machine-data backup " + "after it", + f"`{HOW_TO}` - the whole procedure, restoring included", + ), +)) +def init_command(): + """Create the backup target's bare repository and make it `origin` when there is none.""" + try: + cfg = backup.require_config() + result = backup.init(cfg) + except ChemenuError as exc: + fail(str(exc)) + if result.created_repo: + typer.echo(f"Created the bare repository {result.git_dir}.") + else: + typer.echo(f"{result.git_dir} is already a bare repository - left as it is.") + if result.set_origin: + typer.echo(f"Set origin to {result.git_dir}.") + if result.allowlist_entry: + typer.echo( + "remotes.allowed_push_urls does not list this push target, so `publish` will stop at " + "the Publish-Remote Gate. THE USER CANNOT SEE THIS OUTPUT - show them this section, to " + f"put into {config.LOCAL_SETTINGS_FILENAME} themselves:\n{result.allowlist_entry}" + ) + success("Backup target ready; the next `publish` pushes to it.") + + +@app.command("run") +@cli_contract.record(cli_contract.CommandRecord( + path="backup run", + summary="Mirror this machine's files that are not in git into `//`.", + synopsis=(cli_contract.Variant(usage="backup run"),), + properties=cli_contract.Properties( + effect=cli_contract.Effect.WRITE, + idempotent=cli_contract.Idempotent.YES, + atomic="Per file - each is written to a temporary file and renamed, the manifest last; an " + "interrupted run leaves whole files and the previous manifest", + budget=cli_contract.Budget.COUNTED, + ), + notes=( + "Mirrors `.wikitool-local.json`, `ENVIRONMENT.md` if present, every file in `incoming/` " + "but `.gitkeep`, and the `extra` paths of the `backup` section into " + "`//`, then writes the manifest `.chemenu-backup.json` there: time, " + "stack version, machine name, and per file its relative path, size and sha256.", + "Never mirrors `mcp-upload/`, `reports/`, `.wikitool-tools.json`, `tools/.venv/`, the " + "published skills or `.git/` - unless an `extra` path names one of the first ones. The " + "secrets file lies outside the instance and is never read.", + "Writes and deletes only under `//`. Deletes only a file the previous " + "manifest lists and the instance no longer has; a file there the manifest does not list " + "stays as it is. A file whose size and sha256 match the previous manifest is not " + "rewritten.", + "A symbolic link or junction inside a mirrored directory is skipped and named in the " + "output, never followed.", + "`publish` runs the same step after every push when `machine_data` is true.", + ), + failures=( + _SECTION_FAILURE, + cli_contract.Failure( + cause="`machine_data` is false", + reaction="Show the message to the user; turning it on is their edit of " + "`.wikitool-local.json`", + ), + cli_contract.Failure( + cause="`//` is not empty and has no manifest, or its manifest cannot " + "be read; nothing was written", + reaction="Show the message to the user - the folder is not one this backup wrote. " + "They set another `machine_name` or empty it", + ), + cli_contract.Failure( + cause="The target is not a directory (the synced folder is missing or not mounted), " + "or a write failed", + reaction="Show the message to the user; once the cause is fixed, run `backup run` " + "again", + ), + ), + examples=("tools/wikitool backup run",), + never=( + "Never point `extra` at a path outside the instance to back something else up - the " + "command refuses it, and the secrets belong in no synced folder.", + ), + see_also=( + "`wikitool backup init` - sets up the target's bare repository", + "`wikitool doctor` - the `backup` line reports the age of the last run", + f"`{HOW_TO}` - the whole procedure, restoring included", + ), +)) +def run_command(): + """Mirror this machine's files that are not in git into `//`.""" + try: + cfg = backup.require_config() + if not cfg.machine_data: + fail(f"backup.machine_data is false in {config.LOCAL_SETTINGS_FILENAME} - there is " + "nothing to back up with `backup run`; the user sets it to true by hand") + result = backup.run(cfg) + except ChemenuError as exc: + fail(str(exc)) + except OSError as exc: + fail(f"the machine-data backup failed: {exc}") + success(result.summary()) diff --git a/tools/chemenu/commands/doctor.py b/tools/chemenu/commands/doctor.py index 770dc8b..0015d1a 100644 --- a/tools/chemenu/commands/doctor.py +++ b/tools/chemenu/commands/doctor.py @@ -16,6 +16,7 @@ import shutil import subprocess import sys from dataclasses import dataclass +from pathlib import Path from typing import Optional import typer @@ -51,6 +52,7 @@ def _git(args: list[str]) -> Optional[subprocess.CompletedProcess]: PREFLIGHT_FIX = f"Run tools/preflight.sh (PowerShell 7: {toolpaths.PREFLIGHT_PWSH})" +BACKUP_HOW_TO = "docs/how-to/back-up-to-a-synced-folder.md" def check_python() -> Check: @@ -260,7 +262,8 @@ def check_git_repo() -> list[Check]: Check( "git-remote", "WARN", "No 'origin' remote configured", "A local-only instance is valid - `git remote add origin ` if you want one. " - "Every `publish` needs --no-push until then", + "Every `publish` needs --no-push until then. Without a git server, a synced " + f"folder can serve: {BACKUP_HOW_TO}", ) ) return checks @@ -492,6 +495,77 @@ def check_publish_remotes() -> Check: ) +def check_sync_folder() -> Check: + """Whether the git directory lies in a folder a sync client uploads. Never a + `FAIL`: the instance works there, until the sync client locks, offloads or + conflict-copies a file under `.git` - which is what the `WARN` is for.""" + from chemenu import backup + + result = _git(["rev-parse", "--absolute-git-dir"]) + if result is None or result.returncode != 0 or not result.stdout.strip(): + return Check("sync-folder", "OK", "no git directory to check") + git_dir = result.stdout.strip() + synced = backup.synced_folder_of(Path(git_dir)) + if synced is None: + return Check("sync-folder", "OK", f"{git_dir} lies outside every synced folder this machine names") + return Check( + "sync-folder", "WARN", + f"{git_dir} lies in the synced folder {synced} - a sync client locks files under .git for " + "a moment (which breaks `git gc`), offloads them as placeholders, and makes conflict " + "copies once a second device syncs it", + f"Move the instance out of the synced folder and back it up into it instead: {BACKUP_HOW_TO}. " + "On Windows, check whether Documents or Desktop is redirected into OneDrive (Known Folder " + "Move) - an instance there lies in OneDrive without looking like it", + ) + + +def check_backup() -> Check: + """The `backup` section: absent is `OK` (no backup configured), malformed is + a `FAIL`, and a configured one whose target or `origin` is not set up, or + whose machine data was never backed up, is a `WARN`.""" + from chemenu import backup + + try: + cfg = backup.read_config() + except ChemenuError as exc: + return Check( + "backup", "FAIL", str(exc), + "Fix the backup section as the message says - a broken one is not 'no backup'", + ) + if cfg is None: + return Check("backup", "OK", "absent - no backup section in the settings") + problems, details = [], [f"target {cfg.target}"] + if not backup.is_bare_repo(cfg.git_dir): + problems.append(f"{cfg.git_dir} is not a bare repository") + else: + url = backup.push_url(config.ROOT) + if url is None or not backup.same_location(url, cfg.git_dir, config.ROOT): + problems.append(f"origin pushes to {url or 'nothing'}, not to {cfg.git_dir}") + if cfg.machine_data: + try: + manifest = backup.read_manifest(cfg.machine_dir) + except ChemenuError as exc: + manifest = None + problems.append(str(exc)) + else: + if manifest is None: + problems.append(f"no machine-data backup in {cfg.machine_dir} yet") + else: + details.append( + f"last machine-data backup {backup.age_text(manifest.get('created'))} " + f"({len(manifest['files'])} file(s) in {cfg.machine_dir})" + ) + else: + details.append("machine data not backed up (machine_data is false)") + if problems: + return Check( + "backup", "WARN", "; ".join([*details, *problems]), + f"Run `tools/wikitool backup init`, then `publish` or `tools/wikitool backup run` - " + f"see {BACKUP_HOW_TO}", + ) + return Check("backup", "OK", "; ".join(details)) + + def check_generated_files() -> Check: missing = [ rel_path(path) @@ -950,6 +1024,8 @@ def run_doctor() -> list[Check]: check_environment(), *check_legacy_settings(), check_publish_remotes(), + check_sync_folder(), + check_backup(), check_upload_intake(), check_tasks_provider(), *check_settings(), @@ -1002,6 +1078,14 @@ def run_doctor() -> list[Check]: "Legacy settings (`legacy-settings`): one `FAIL` for each per-feature settings file of " "an earlier version (telemetry, remotes, upload, task tracker) that is still there. Nothing reads them any more, so the settings they hold are not in effect; " "the fix is `tools/wikitool config migrate`.", + "Synced folder (`sync-folder`): a `WARN` when the git directory, links and junctions " + "resolved, lies under a folder named by `OneDrive`, `OneDriveCommercial` or " + "`OneDriveConsumer` (compared without regard to case on Windows) or under " + "`~/Library/CloudStorage/` (macOS); `OK` otherwise, never a `FAIL`.", + "Backup (`backup`): the `backup` section absent is `OK`, malformed is a `FAIL`. " + "Configured, a `WARN` when `/git` is not a bare repository, `origin` does not " + "push there, or - with `machine_data` true - no machine-data backup exists yet; " + "otherwise `OK` with the age of the last machine-data backup.", "MCP `submit` tool: whether the `upload` section of the settings is present, absent or " "malformed, its limits, and how many submissions wait in `mcp-upload/`. Absent is `OK` " "and means the write path does not exist at all; malformed is a `FAIL`.", @@ -1059,6 +1143,7 @@ def run_doctor() -> list[Check]: "`docs/reference/configuration.md` - the per-checkout configuration files and the tiered " "settings", "`docs/explanation/telemetry-and-evaluation.md` - telemetry state and caps", + f"`{BACKUP_HOW_TO}` - what `sync-folder` and `backup` point to", "`instructions/session-setup.md` - setting `WIKITOOL_SESSION_ID`", ), )) diff --git a/tools/chemenu/commands/git_publish.py b/tools/chemenu/commands/git_publish.py index 5d22024..8cf827a 100644 --- a/tools/chemenu/commands/git_publish.py +++ b/tools/chemenu/commands/git_publish.py @@ -63,7 +63,7 @@ from typing import NamedTuple, Optional import typer from chemenu import cli_contract, config, config_shapes, toolpaths -from chemenu.commands._util import fail, load_settings, needs_clearance, refuse_leftover, success +from chemenu.commands._util import console, fail, load_settings, needs_clearance, refuse_leftover, success from chemenu.telemetry import emit # "Farza's Rule" from the Mass-Update Gate concept: halt and request @@ -159,6 +159,45 @@ def publish_remote_refusal(remote: str, branch: str) -> Optional[str]: ) +# --- Machine-data backup ------------------------------------------------------ +# +# With a `backup` section whose `machine_data` is true, a publish that pushed +# also mirrors this machine's files that are not in git (`chemenu.backup`). By +# then the commit and the push have happened, so a failing backup is a WARN and +# exit 0: exit 1 would read as "publish failed" and invite a second publish, +# which cannot fix a target that is not writable - `backup run` after the cause +# is fixed does. + + +def read_backup_config(): + """The `backup` section, None when absent; exit 1 for a broken one.""" + from chemenu import backup + from chemenu.errors import ChemenuError + + try: + return backup.read_config() + except ChemenuError as exc: + fail(str(exc)) + + +def back_up_machine_data(cfg) -> None: + from rich.markup import escape + + from chemenu import backup + from chemenu.errors import ChemenuError + + try: + result = backup.run(cfg) + except (ChemenuError, OSError) as exc: + console.print( + f"[yellow]WARN[/yellow] The machine-data backup failed: {escape(str(exc))}\n" + " The commit and the push stand. Show this to the user; once the cause is fixed, " + "`tools/wikitool backup run` repeats the backup - another `publish` does not." + ) + return + typer.echo(result.summary()) + + # The status letters `git diff --cached --raw` emits under `--no-renames` (renames and copies # are switched off, so neither R nor C can appear). Anything else - M, and T for a type change - # is a modification. @@ -1568,6 +1607,13 @@ def commit_regenerated(remote: str, branch: str) -> None: "`git interpret-trailers` decides whether there is such a block, and the list moves " "only when git reads the same trailers from the result; otherwise it is appended at " "the end. `--message` is not part of any gate token.", + "Machine-data backup: with a `backup` section in `.wikitool-local.json` whose " + "`machine_data` is true, a call that pushed then mirrors this machine's files that are " + "not in git into `//` - the same step as `backup run`. A failure there " + "prints a `WARN` line with the reason and still exits 0, since the commit and the push " + "stand. No backup with `--no-push`, after a failed push, or when there was nothing to " + "push. A malformed `backup` section stops the call with exit 1 before anything is " + "committed, `--no-push` included.", "After a successful commit or push whose changed files include `tools/`, `types/`, " "`instructions/`, `AGENTS.md` or a path ending in `CONTRACT.md`, prints one reminder " "line: the phase past this point (an issue-body rewrite, `docs/` staleness, a " @@ -1604,11 +1650,18 @@ def commit_regenerated(remote: str, branch: str) -> None: "from the gate's own refusal output", ), cli_contract.Failure( - cause="The settings files do not pass their check, or `remotes.allowed_push_urls` is " - "not a list of strings", + cause="The settings files do not pass their check, `remotes.allowed_push_urls` is " + "not a list of strings, or the `backup` section is malformed; nothing was committed", reaction="Show the error to the user and stop - a malformed section is not " "permission, and fixing it is theirs to do", ), + cli_contract.Failure( + cause="The machine-data backup after the push failed: a `WARN` line names the reason, " + "and the commit and the push stand", + reaction="Show the `WARN` line to the user. Do not publish again - once the cause is " + "fixed, `tools/wikitool backup run` repeats the backup", + code=0, + ), cli_contract.Failure( cause="The remotes settings file of an earlier version is still there", reaction="Show the output to the user verbatim and stop; `tools/wikitool config " @@ -1656,6 +1709,7 @@ def commit_regenerated(remote: str, branch: str) -> None: "`wikitool sync` - the same reconcile on its own, without committing or pushing", "`instructions/gates.md` - the gate procedure", "`instructions/setup-instance.md` - the first publish of a new instance", + "`wikitool backup run` - the machine-data backup on its own", ), )) def publish_command( @@ -1703,6 +1757,10 @@ def publish_command( if yes: fail(YES_REMOVED_MESSAGE) + # Read before anything is committed, `--no-push` included: a broken section + # stops here rather than turning into a skipped backup after the push. + backup_cfg = read_backup_config() + paths = list(path or []) # What is staged and committed: `paths` plus any uncommitted generated file outside them. # The gate token and its re-run line keep `paths`. @@ -1872,5 +1930,7 @@ def publish_command( }, ) success(f"Published changes to {remote}/{branch}." if push else "Committed changes (not pushed).") + if push and backup_cfg is not None and backup_cfg.machine_data: + back_up_machine_data(backup_cfg) if touches_stack_machinery(changed_files): typer.echo(STACK_MACHINERY_NOTE) diff --git a/tools/chemenu/config_shapes.py b/tools/chemenu/config_shapes.py index 61dbe88..4f3d8c5 100644 --- a/tools/chemenu/config_shapes.py +++ b/tools/chemenu/config_shapes.py @@ -31,6 +31,8 @@ every tool call without the venv. from __future__ import annotations import json +import ntpath +import posixpath import re from dataclasses import dataclass, field as dc_field from typing import Any, Callable, Optional @@ -576,11 +578,101 @@ FETCH = Shape( "of that host the same way.", ) +def strict_bool(value: Any) -> bool: + if not isinstance(value, bool): + raise TypeError("not a boolean") + return value + + +_MACHINE_NAME_INVALID = re.compile(r"[^A-Za-z0-9_-]") +# The one subdirectory name of a backup target that is not a machine's. +BACKUP_GIT_DIR = "git" + + +def machine_name(value: str) -> str: + """`value` cleaned to a directory name of a backup target: every character + outside `[A-Za-z0-9_-]` becomes `-`, and leading and trailing `-` go.""" + return _MACHINE_NAME_INVALID.sub("-", value).strip("-") + + +def machine_name_problem(value: str) -> Optional[str]: + cleaned = machine_name(value) + if not cleaned: + return "is empty once cleaned to letters, digits, `-` and `_`" + if cleaned.casefold() == BACKUP_GIT_DIR: + return f"cleans to {cleaned!r}, the name of the target's git directory" + return None + + +def absolute_path_problem(value: str) -> Optional[str]: + # Either platform's form, so a record's example holds on both; whether the + # path is absolute *on this machine* is `chemenu.backup.read_config`'s check. + return None if ntpath.isabs(value) or posixpath.isabs(value) else "must be an absolute path" + + +def relative_paths_problem(value: list[str]) -> Optional[str]: + for entry in value: + if not entry.strip(): + return "must not contain an empty path" + if ntpath.isabs(entry) or posixpath.isabs(entry): + return f"holds {entry!r}, an absolute path - each entry is relative to the instance" + return None + + +BACKUP = Shape( + key="backup", + title="Backup to a synced folder", + purpose="Backs this checkout up into a folder a sync client (OneDrive and the like) uploads, " + "without the working tree living in it: a bare repository under `/git` as the push " + "target of `publish`, and a mirror of the files of this machine that are not in git under " + "`//`.", + fields=( + Field( + "target", "string", "The backup folder: an absolute path outside the instance. " + "`backup init` creates `/git` in it, `backup run` writes only to " + "`//`.", + required=True, json_type="string", coerce=exact_str, check=absolute_path_problem, + tier=CHECKOUT, gate=True, + ), + Field( + "machine_data", "boolean", "Mirror this machine's files after every `publish` that " + "pushed, and with `backup run`: `.wikitool-local.json`, `ENVIRONMENT.md`, what waits " + "in `incoming/`, and `extra`.", + default=False, json_type="boolean", coerce=strict_bool, tier=CHECKOUT, gate=True, + ), + Field( + "machine_name", "string", "The name of this machine's folder under the target. " + "Every character outside letters, digits, `-` and `_` becomes `-`, and leading and " + "trailing `-` are dropped; a name that is empty afterwards, or `git` in any case, is " + "an error.", + default_text="the host name", json_type="string", coerce=exact_str, + check=machine_name_problem, tier=CHECKOUT, + ), + Field( + "extra", "array of strings", "Further files or directories to mirror, relative to " + "the instance. One that resolves outside the instance, or into `.git/`, is an error.", + default=[], json_type="array", coerce=str_list, check=relative_paths_problem, + tier=CHECKOUT, gate=True, + ), + ), + examples=({ + "target": "C:\\Users\\\\OneDrive - \\Chemenu-Backup", + "machine_data": True, + "machine_name": "WORK-LAPTOP", + "extra": [".claude/settings.local.json"], + },), + absent="No backup: `publish` behaves as without it, and `backup init` and `backup run` stop " + "with exit 1 naming the section.", + malformed="An error. `publish` stops with exit 1 before it commits anything, `backup init` and " + "`backup run` stop with exit 1, and `doctor` reports `FAIL`. A broken section is never read as " + "\"no backup\".", +) + # The sections of the tiered settings files, `wikitool.json` and # `.wikitool-local.json` (`chemenu.settings`), read at call time so a test can # stand in a section of its own. SETTINGS_SECTIONS: tuple[Shape, ...] = ( - TELEMETRY, REMOTES, UPLOAD, TASKS, SUPERPRODUCTIVITY, CALDAV, SITES, FETCH, + TELEMETRY, REMOTES, UPLOAD, TASKS, SUPERPRODUCTIVITY, CALDAV, SITES, FETCH, BACKUP, ) # The sections `WIKITOOL_TASKS_CONFIG` replaces as a whole: a profile names a diff --git a/tools/chemenu/reference_doc.py b/tools/chemenu/reference_doc.py index 9567994..9e54055 100644 --- a/tools/chemenu/reference_doc.py +++ b/tools/chemenu/reference_doc.py @@ -70,6 +70,7 @@ CONFIG_REGIONS = ( ("reference-config-upload", "UPLOAD"), ("reference-config-sites", "SITES"), ("reference-config-fetch", "FETCH"), + ("reference-config-backup", "BACKUP"), ) _SLUG_RE = re.compile(r"[^a-z0-9]+") diff --git a/tools/chemenu/tests/conftest.py b/tools/chemenu/tests/conftest.py index a62f3f3..0f588a5 100644 --- a/tools/chemenu/tests/conftest.py +++ b/tools/chemenu/tests/conftest.py @@ -39,6 +39,10 @@ _WIKITOOL_ENV = ( "CHEMENU_PREFLIGHT_POLICY", "CHEMENU_PREFLIGHT_MARKED", "CHEMENU_PREFLIGHT_MOVE_FAIL", + # Where a sync client keeps its folder - `doctor`'s sync-folder check reads them. + "OneDrive", + "OneDriveCommercial", + "OneDriveConsumer", ) + tuple(var for var, _harness in HARNESS_ENV_VARS) # Environment git reads for identity or for where its repo lives. A stray diff --git a/tools/chemenu/tests/test_backup.py b/tools/chemenu/tests/test_backup.py new file mode 100644 index 0000000..f0c8559 --- /dev/null +++ b/tools/chemenu/tests/test_backup.py @@ -0,0 +1,562 @@ +"""`backup init`, `backup run`, the machine-data step of `publish`, and the two +`doctor` lines - against a real checkout and a target directory in `tmp_path`. + +The target's path carries a space on purpose: the folder a sync client keeps on +Windows is `OneDrive - `, and every path that reaches git or the file +system has to survive that.""" +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + +import pytest +import typer +from typer.testing import CliRunner + +from chemenu import backup, config, config_shapes, settings +from chemenu.cli import app +from chemenu.commands import doctor +from chemenu.commands.git_publish import publish_command + +runner = CliRunner() + + +def _git(root: Path, *args: str) -> str: + result = subprocess.run(["git", *args], cwd=root, capture_output=True, text=True) + assert result.returncode == 0, result.stderr + return result.stdout.strip() + + +def _write_local(root: Path, **sections) -> None: + (root / config.LOCAL_SETTINGS_FILENAME).write_text( + json.dumps({"schema": 1, **sections}, indent=2), encoding="utf-8") + + +@pytest.fixture +def instance(tmp_path, monkeypatch): + """A checkout with one commit and no remote, and an empty synced folder + beside it - whose name has a space in it.""" + root = tmp_path / "instance" + root.mkdir() + _git(root, "init", "-b", "main") + _git(root, "config", "user.name", "Test") + _git(root, "config", "user.email", "test@example.com") + (root / "kb").mkdir() + (root / "incoming").mkdir() + (root / "incoming" / ".gitkeep").write_text("", encoding="utf-8") + (root / ".gitignore").write_text( + "/incoming/*\n!/incoming/.gitkeep\n/.wikitool-local.json\n/ENVIRONMENT.md\n/mcp-upload/\n" + "/reports/\n/.wikitool-tools.json\n", encoding="utf-8") + (root / "README.md").write_text("init\n", encoding="utf-8") + _git(root, "add", "-A") + _git(root, "commit", "-m", "init") + synced = tmp_path / "OneDrive - Example" + synced.mkdir() + monkeypatch.setattr(config, "ROOT", root) + monkeypatch.setenv("WIKITOOL_SESSION_ID", "test-session") + return root + + +def _target(root: Path) -> Path: + return root.parent / "OneDrive - Example" / "Chemenu-Backup" + + +def _configure(root: Path, **overrides) -> Path: + target = _target(root) + section = {"target": str(target), "machine_data": True, "machine_name": "WORK-LAPTOP"} + section.update(overrides) + _write_local(root, backup=section) + return target + + +def _snapshot(directory: Path) -> dict: + """Every file below `directory` with its bytes - what "unchanged" is measured against.""" + return {p.relative_to(directory).as_posix(): p.read_bytes() + for p in sorted(directory.rglob("*")) if p.is_file()} + + +def _mirrored(target: Path, machine: str = "WORK-LAPTOP") -> set[str]: + return set(_snapshot(target / machine)) + + +def _flat(text: str) -> str: + """Output with Rich's line wrapping undone.""" + return " ".join(text.split()) + + +def _publish(**overrides): + kwargs = dict(message="change", push=True, confirm=None, confirm_rebase=None, yes=False, + threshold=10, remote="origin", branch="main", path=None) + kwargs.update(overrides) + publish_command(**kwargs) + + +# --- the section ------------------------------------------------------------- + + +def test_no_section_reads_as_none_and_the_commands_name_it(instance): + assert backup.read_config() is None + for command in ("init", "run"): + result = runner.invoke(app, ["backup", command]) + assert result.exit_code == 1 + assert "no backup section" in _flat(result.output) + + +@pytest.mark.parametrize("name", ["", "...", " ", "git", "GIT", ".git", "-Git-"]) +def test_a_machine_name_that_cleans_to_nothing_or_git_is_an_error(instance, name): + _configure(instance, machine_name=name) + with pytest.raises(settings.ValidationError, match="machine_name"): + backup.read_config() + + +def test_a_machine_name_is_cleaned_to_a_directory_name(instance): + _configure(instance, machine_name="Work Laptop.corp") + assert backup.read_config().machine_name == "Work-Laptop-corp" + + +def test_without_a_machine_name_the_host_name_is_used(instance, monkeypatch): + monkeypatch.setattr(backup.socket, "gethostname", lambda: "host.example") + _configure(instance) + data = json.loads((instance / config.LOCAL_SETTINGS_FILENAME).read_text(encoding="utf-8")) + del data["backup"]["machine_name"] + (instance / config.LOCAL_SETTINGS_FILENAME).write_text(json.dumps(data), encoding="utf-8") + assert backup.read_config().machine_name == "host-example" + + +def test_a_target_inside_the_instance_is_an_error(instance): + _configure(instance, target=str(instance / "backup")) + with pytest.raises(settings.ValidationError, match="inside the instance"): + backup.read_config() + + +def test_a_relative_target_is_an_error(instance): + _configure(instance, target="backup") + with pytest.raises(settings.ValidationError, match="absolute"): + backup.read_config() + + +@pytest.mark.parametrize("field, value", [("machine_data", "yes"), ("extra", "a")]) +def test_a_field_of_the_wrong_type_is_an_error(instance, field, value): + _configure(instance, **{field: value}) + with pytest.raises(settings.ValidationError, match=field): + backup.read_config() + + +def test_the_gate_fields_are_refused_by_config_set(instance): + for name in ("target", "machine_data", "extra"): + assert config_shapes.BACKUP.field(name).user_only + result = runner.invoke(app, ["config", "set", "backup.target", "/tmp/x"]) + assert result.exit_code == 42 + + +# --- backup init ------------------------------------------------------------- + + +def test_init_creates_a_bare_repository_and_makes_it_origin(instance): + target = _configure(instance) + result = runner.invoke(app, ["backup", "init"]) + assert result.exit_code == 0, result.output + assert backup.is_bare_repo(target / "git") + assert _git(instance, "remote", "get-url", "--push", "origin") == str(target / "git") + + before = _snapshot(target) + again = runner.invoke(app, ["backup", "init"]) + assert again.exit_code == 0, again.output + assert "left as it is" in _flat(again.output) + assert _snapshot(target) == before + assert _git(instance, "remote", "get-url", "--push", "origin") == str(target / "git") + + +def test_init_into_an_empty_directory_uses_it(instance): + target = _configure(instance) + (target / "git").mkdir(parents=True) + assert runner.invoke(app, ["backup", "init"]).exit_code == 0 + assert backup.is_bare_repo(target / "git") + + +def test_init_leaves_an_origin_that_points_elsewhere_and_changes_nothing(instance, tmp_path): + target = _configure(instance) + _git(instance, "remote", "add", "origin", "https://example.org/other.git") + result = runner.invoke(app, ["backup", "init"]) + assert result.exit_code == 1 + assert "https://example.org/other.git" in _flat(result.output) + assert str(target / "git") in _flat(result.output) + assert _git(instance, "remote", "get-url", "--push", "origin") == "https://example.org/other.git" + assert not target.exists() + + +def test_init_refuses_a_git_folder_that_is_not_a_bare_repository(instance): + target = _configure(instance) + (target / "git").mkdir(parents=True) + (target / "git" / "notes.txt").write_text("mine\n", encoding="utf-8") + before = _snapshot(target) + result = runner.invoke(app, ["backup", "init"]) + assert result.exit_code == 1 + assert "not a bare git repository" in _flat(result.output) + assert _snapshot(target) == before + assert subprocess.run(["git", "remote", "get-url", "origin"], cwd=instance, + capture_output=True).returncode != 0 + + +def test_init_prints_the_allowlist_entry_and_never_writes_it(instance): + target = _configure(instance) + data = json.loads((instance / config.LOCAL_SETTINGS_FILENAME).read_text(encoding="utf-8")) + data["remotes"] = {"allowed_push_urls": ["https://example.org/kb.git"]} + (instance / config.LOCAL_SETTINGS_FILENAME).write_text(json.dumps(data), encoding="utf-8") + before = (instance / config.LOCAL_SETTINGS_FILENAME).read_bytes() + + result = runner.invoke(app, ["backup", "init"]) + assert result.exit_code == 0, result.output + entry = json.loads(result.output[result.output.index("{"):result.output.rindex("}") + 1]) + assert entry == {"remotes": {"allowed_push_urls": ["https://example.org/kb.git", str(target / "git")]}} + assert (instance / config.LOCAL_SETTINGS_FILENAME).read_bytes() == before + + +# --- backup run -------------------------------------------------------------- + + +def _fill(root: Path) -> None: + (root / "ENVIRONMENT.md").write_text("# env\n", encoding="utf-8") + (root / "incoming" / "paper.pdf").write_bytes(b"%PDF-1.7 fixture") + (root / "incoming" / "old").mkdir() + (root / "incoming" / "old" / "note.md").write_text("note\n", encoding="utf-8") + for relative in ("mcp-upload/sub-1/x.md", "reports/lint.md", ".wikitool-tools.json"): + (root / relative).parent.mkdir(parents=True, exist_ok=True) + (root / relative).write_text("not mirrored\n", encoding="utf-8") + (root / ".claude").mkdir() + (root / ".claude" / "settings.local.json").write_text("{}\n", encoding="utf-8") + + +def test_run_mirrors_exactly_the_machine_data(instance): + target = _configure(instance, extra=[".claude/settings.local.json"]) + target.mkdir(parents=True) + _fill(instance) + result = runner.invoke(app, ["backup", "run"]) + assert result.exit_code == 0, result.output + assert _mirrored(target) == { + ".wikitool-local.json", "ENVIRONMENT.md", "incoming/paper.pdf", "incoming/old/note.md", + ".claude/settings.local.json", backup.MANIFEST_NAME, + } + machine = target / "WORK-LAPTOP" + assert (machine / "incoming" / "paper.pdf").read_bytes() == b"%PDF-1.7 fixture" + manifest = json.loads((machine / backup.MANIFEST_NAME).read_text(encoding="utf-8")) + assert manifest["machine"] == "WORK-LAPTOP" + assert {entry["path"] for entry in manifest["files"]} == _mirrored(target) - {backup.MANIFEST_NAME} + assert all(len(entry["sha256"]) == 64 and entry["size"] >= 0 for entry in manifest["files"]) + + +def test_run_never_mirrors_git_even_through_extra(instance): + _configure(instance, extra=[".git"]) + with pytest.raises(settings.ValidationError, match=r"\.git"): + backup.read_config() + target = _configure(instance, extra=["kb"]) + target.mkdir(parents=True) + (instance / "kb" / ".git").mkdir() + (instance / "kb" / ".git" / "HEAD").write_text("ref\n", encoding="utf-8") + (instance / "kb" / "page.md").write_text("page\n", encoding="utf-8") + assert runner.invoke(app, ["backup", "run"]).exit_code == 0 + assert "kb/page.md" in _mirrored(target) + assert not any(".git/" in path for path in _mirrored(target)) + + +def test_a_second_run_removes_what_left_incoming_and_keeps_a_foreign_file(instance): + target = _configure(instance) + target.mkdir(parents=True) + _fill(instance) + assert runner.invoke(app, ["backup", "run"]).exit_code == 0 + foreign = target / "WORK-LAPTOP" / "kept-by-hand.txt" + foreign.write_bytes(b"mine") + + (instance / "incoming" / "paper.pdf").unlink() + result = runner.invoke(app, ["backup", "run"]) + assert result.exit_code == 0, result.output + assert "1 removed" in _flat(result.output) + assert not (target / "WORK-LAPTOP" / "incoming" / "paper.pdf").exists() + assert foreign.read_bytes() == b"mine" + + +def test_an_unchanged_file_is_not_rewritten(instance): + target = _configure(instance) + target.mkdir(parents=True) + _fill(instance) + assert runner.invoke(app, ["backup", "run"]).exit_code == 0 + copy = target / "WORK-LAPTOP" / "incoming" / "paper.pdf" + stamp = copy.stat().st_mtime_ns + os.utime(copy, ns=(stamp - 10**9, stamp - 10**9)) + result = runner.invoke(app, ["backup", "run"]) + assert "0 written" in _flat(result.output) + assert copy.stat().st_mtime_ns == stamp - 10**9 + + +def test_run_refuses_a_non_empty_folder_without_a_manifest(instance): + target = _configure(instance) + (target / "WORK-LAPTOP").mkdir(parents=True) + (target / "WORK-LAPTOP" / "somebody-elses.txt").write_text("x\n", encoding="utf-8") + _fill(instance) + before = _snapshot(target) + result = runner.invoke(app, ["backup", "run"]) + assert result.exit_code == 1 + assert "no .chemenu-backup.json" in _flat(result.output) + assert _snapshot(target) == before + + +def test_run_writes_nothing_outside_its_own_folder(instance): + target = _configure(instance) + target.mkdir(parents=True) + (target / "OTHER-PC").mkdir() + (target / "OTHER-PC" / "a.txt").write_text("other\n", encoding="utf-8") + (target / "git").mkdir() + (target / "git" / "b").write_text("repo\n", encoding="utf-8") + before = {k: v for k, v in _snapshot(target).items()} + _fill(instance) + assert runner.invoke(app, ["backup", "run"]).exit_code == 0 + after = _snapshot(target) + assert {k: v for k, v in after.items() if not k.startswith("WORK-LAPTOP/")} == before + + +def test_a_manifest_entry_that_leaves_the_folder_is_refused(instance): + target = _configure(instance) + (target / "WORK-LAPTOP").mkdir(parents=True) + (target / "victim.txt").write_text("keep\n", encoding="utf-8") + (target / "WORK-LAPTOP" / backup.MANIFEST_NAME).write_text(json.dumps( + {"schema": 1, "files": [{"path": "../victim.txt", "size": 5, "sha256": "0" * 64}]}), + encoding="utf-8") + assert runner.invoke(app, ["backup", "run"]).exit_code == 1 + assert (target / "victim.txt").read_text(encoding="utf-8") == "keep\n" + + +def test_an_extra_path_outside_the_instance_is_refused_before_anything_is_written(instance, tmp_path): + target = _configure(instance, extra=["../outside.txt"]) + target.mkdir(parents=True) + (tmp_path / "outside.txt").write_text("secret\n", encoding="utf-8") + result = runner.invoke(app, ["backup", "run"]) + assert result.exit_code == 1 + assert "outside the instance" in _flat(result.output) + assert not (target / "WORK-LAPTOP").exists() + + +@pytest.mark.skipif(not hasattr(os, "symlink") or os.name == "nt", reason="symlinks") +def test_an_extra_symlink_to_outside_is_refused(instance, tmp_path): + (tmp_path / "secrets").mkdir() + (tmp_path / "secrets" / "secrets.json").write_text("{}\n", encoding="utf-8") + (instance / "linked").symlink_to(tmp_path / "secrets") + target = _configure(instance, extra=["linked"]) + target.mkdir(parents=True) + assert runner.invoke(app, ["backup", "run"]).exit_code == 1 + assert not (target / "WORK-LAPTOP").exists() + + +@pytest.mark.skipif(not hasattr(os, "symlink") or os.name == "nt", reason="symlinks") +def test_a_symlink_in_incoming_is_skipped_and_named(instance, tmp_path): + (tmp_path / "elsewhere.txt").write_text("not mine\n", encoding="utf-8") + (instance / "incoming" / "link.txt").symlink_to(tmp_path / "elsewhere.txt") + target = _configure(instance) + target.mkdir(parents=True) + result = runner.invoke(app, ["backup", "run"]) + assert result.exit_code == 0, result.output + assert "incoming/link.txt (a link - not followed)" in _flat(result.output) + assert "incoming/link.txt" not in _mirrored(target) + + +def test_run_refuses_while_machine_data_is_off(instance): + _configure(instance, machine_data=False) + result = runner.invoke(app, ["backup", "run"]) + assert result.exit_code == 1 + assert "machine_data is false" in _flat(result.output) + + +def test_run_refuses_a_missing_target(instance): + target = _configure(instance) + result = runner.invoke(app, ["backup", "run"]) + assert result.exit_code == 1 + assert "not a directory" in _flat(result.output) + assert not target.exists() + + +# --- publish ----------------------------------------------------------------- + + +def _ready(root: Path, **overrides) -> Path: + target = _configure(root, **overrides) + assert runner.invoke(app, ["backup", "init"]).exit_code == 0 + return target + + +def test_publish_pushes_to_the_target_and_backs_the_machine_data_up(instance): + target = _ready(instance, extra=[".claude/settings.local.json"]) + _fill(instance) + (instance / "kb" / "page.md").write_text("page\n", encoding="utf-8") + _publish() + assert _git(target / "git", "rev-parse", "main") == _git(instance, "rev-parse", "HEAD") + assert _mirrored(target) == { + ".wikitool-local.json", "ENVIRONMENT.md", "incoming/paper.pdf", "incoming/old/note.md", + ".claude/settings.local.json", backup.MANIFEST_NAME, + } + + +def test_publish_without_machine_data_backs_nothing_up(instance): + target = _ready(instance, machine_data=False) + (instance / "kb" / "page.md").write_text("page\n", encoding="utf-8") + _publish() + assert _git(target / "git", "rev-parse", "main") == _git(instance, "rev-parse", "HEAD") + assert not (target / "WORK-LAPTOP").exists() + + +def test_a_failed_machine_backup_is_a_warning_after_the_push(instance, capsys): + target = _ready(instance) + (target / "WORK-LAPTOP").mkdir() + (target / "WORK-LAPTOP" / "somebody-elses.txt").write_text("x\n", encoding="utf-8") + (instance / "kb" / "page.md").write_text("page\n", encoding="utf-8") + _publish() # no typer.Exit: exit 0 + out = capsys.readouterr().out + assert "WARN" in out and "backup run" in out + assert _git(target / "git", "rev-parse", "main") == _git(instance, "rev-parse", "HEAD") + + +def test_an_unwritable_target_is_a_warning_after_the_push(instance, capsys, monkeypatch): + target = _ready(instance) + + def refuse(*args, **kwargs): + raise PermissionError(13, "Permission denied", str(target)) + + monkeypatch.setattr(backup, "_copy_atomic", refuse) + (instance / "kb" / "page.md").write_text("page\n", encoding="utf-8") + _publish() + out = capsys.readouterr().out + assert "WARN" in out and "Permission denied" in out + assert _git(target / "git", "rev-parse", "main") == _git(instance, "rev-parse", "HEAD") + + +def test_publish_no_push_backs_nothing_up(instance): + target = _ready(instance) + (instance / "kb" / "page.md").write_text("page\n", encoding="utf-8") + _publish(push=False) + assert not (target / "WORK-LAPTOP").exists() + + +def test_a_malformed_section_stops_publish_before_the_commit(instance): + _configure(instance, machine_name="git") + (instance / "kb" / "page.md").write_text("page\n", encoding="utf-8") + head = _git(instance, "rev-parse", "HEAD") + for push in (True, False): + with pytest.raises(typer.Exit) as exc: + _publish(push=push) + assert exc.value.exit_code == 1 + assert _git(instance, "rev-parse", "HEAD") == head + + +# --- doctor ------------------------------------------------------------------ + + +def _check(name: str): + return next(check for check in doctor.run_doctor() if check.name == name) + + +@pytest.fixture +def doctor_instance(instance, monkeypatch): + # `run_doctor` runs every check; the two under test only need the checkout. + for name in ("check_skills", "check_structure", "check_personalization", "check_conventions", + "check_generated_files", "check_tool_paths"): + monkeypatch.setattr(doctor, name, lambda: doctor.Check(name, "OK", "stubbed")) + return instance + + +@pytest.mark.parametrize("var", backup.SYNC_ENV_VARS) +def test_sync_folder_warns_under_a_onedrive_folder(doctor_instance, monkeypatch, var): + monkeypatch.setenv(var, str(doctor_instance.parent)) + check = _check("sync-folder") + assert check.status == "WARN" + assert "back-up-to-a-synced-folder.md" in check.fix and "Known Folder Move" in check.fix + + +def test_sync_folder_warns_under_cloud_storage(doctor_instance, tmp_path, monkeypatch): + cloud = tmp_path / "home" / "Library" / "CloudStorage" + cloud.mkdir(parents=True) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + moved = cloud / "OneDrive-Example" / "instance" + moved.parent.mkdir() + doctor_instance.rename(moved) + monkeypatch.setattr(config, "ROOT", moved) + assert _check("sync-folder").status == "WARN" + + +@pytest.mark.skipif(not hasattr(os, "symlink") or os.name == "nt", reason="symlinks") +def test_sync_folder_sees_through_a_symlink(doctor_instance, tmp_path, monkeypatch): + synced = tmp_path / "OneDrive - Example" + real = synced / "instance" + doctor_instance.rename(real) + (tmp_path / "via-link").symlink_to(real) + monkeypatch.setattr(config, "ROOT", tmp_path / "via-link") + monkeypatch.setenv("OneDrive", str(synced)) + assert _check("sync-folder").status == "WARN" + + +def test_sync_folder_is_ok_elsewhere_and_never_sets_the_exit_code(doctor_instance, tmp_path, monkeypatch): + monkeypatch.setenv("OneDrive", str(tmp_path / "OneDrive - Example")) + assert _check("sync-folder").status == "OK" + + +def test_sync_folder_compares_without_case_on_windows(doctor_instance, monkeypatch): + monkeypatch.setenv("OneDrive", str(doctor_instance.parent).upper()) + monkeypatch.setattr(backup, "IGNORE_CASE", False) + assert _check("sync-folder").status == "OK" + monkeypatch.setattr(backup, "IGNORE_CASE", True) + assert _check("sync-folder").status == "WARN" + + +def test_backup_check_absent_is_ok(doctor_instance): + check = _check("backup") + assert (check.status, check.detail) == ("OK", "absent - no backup section in the settings") + + +def test_backup_check_fails_on_a_malformed_section(doctor_instance): + _configure(doctor_instance, machine_name="GIT") + check = _check("backup") + assert check.status == "FAIL" and "machine_name" in check.detail + + +def test_backup_check_warns_until_init_and_the_first_run(doctor_instance): + target = _configure(doctor_instance) + check = _check("backup") + assert check.status == "WARN" and "not a bare repository" in check.detail + + assert runner.invoke(app, ["backup", "init"]).exit_code == 0 + check = _check("backup") + assert check.status == "WARN" and "no machine-data backup" in check.detail + + assert runner.invoke(app, ["backup", "run"]).exit_code == 0 + check = _check("backup") + assert check.status == "OK" and "last machine-data backup" in check.detail + assert str(target) in check.detail + + +def test_backup_check_warns_on_an_origin_elsewhere(doctor_instance): + target = _configure(doctor_instance, machine_data=False) + subprocess.run(["git", "init", "--bare", "-q", "-b", "main", str(target / "git")], check=True) + _git(doctor_instance, "remote", "add", "origin", "https://example.org/other.git") + check = _check("backup") + assert check.status == "WARN" and "https://example.org/other.git" in check.detail + + +def test_a_warning_from_either_check_leaves_doctors_exit_code_alone(doctor_instance, monkeypatch): + monkeypatch.setenv("OneDrive", str(doctor_instance.parent)) + _configure(doctor_instance) + checks = {check.name: check.status for check in doctor.run_doctor()} + assert checks["sync-folder"] == "WARN" and checks["backup"] == "WARN" + assert all(status != "FAIL" for name, status in checks.items() if name in ("sync-folder", "backup")) + + +def test_the_git_remote_warning_names_the_how_to(doctor_instance): + assert "back-up-to-a-synced-folder.md" in _check("git-remote").fix + + +def test_age_text(): + from datetime import datetime, timezone + + now = datetime(2026, 10, 12, 12, 0, tzinfo=timezone.utc) + assert backup.age_text("2026-10-12T11:59:30+00:00", now) == "less than a minute old" + assert backup.age_text("2026-10-12T09:00:00+00:00", now) == "3 h old" + assert backup.age_text("2026-10-10T12:00:00+00:00", now) == "2 d old" + assert backup.age_text("garbage", now) == "of unknown age" diff --git a/tools/chemenu/tests/test_config_shapes.py b/tools/chemenu/tests/test_config_shapes.py index 4c75aa8..7ff0d63 100644 --- a/tools/chemenu/tests/test_config_shapes.py +++ b/tools/chemenu/tests/test_config_shapes.py @@ -79,6 +79,7 @@ def test_the_telemetry_policy_reads_the_record(): def test_one_record_per_section(): assert [shape.key for shape in config_shapes.SETTINGS_SECTIONS] == [ "telemetry", "remotes", "upload", "tasks", "superproductivity", "caldav", "sites", "fetch", + "backup", ]