From e4b2b6d9b1869e9062bcdaff95161b4f888eae0d Mon Sep 17 00:00:00 2001 From: Torben Nehmer Date: Thu, 1 Oct 2026 05:52:55 +0200 Subject: [PATCH] feat: preflight - prerequisites checked and tool paths recorded before wikitool runs; launcher refuses without it (#151, POSIX half) Files changed: - .claude/settings.json - .gitea/workflows/ci.yml - .gitea/workflows/nightly.yml - .gitea/workflows/release.yml - .gitea/workflows/tracker-live.yml - .github/hooks/wiki-trace.json - .gitignore - .vibe/hooks.toml - AGENTS.md - CHANGES.md - EVALS.md - INSTALL.md - README.md - VERSION - instructions/bootstrap.md - instructions/preflight.md - instructions/setup-instance.md - instructions/upgrade-instance.md - tools/CONTRACT.md - tools/README.md - tools/chemenu/cli.py - tools/chemenu/commands/dist_cmd.py - tools/chemenu/commands/docs_verify.py - tools/chemenu/commands/doctor.py - tools/chemenu/commands/git_publish.py - tools/chemenu/commands/migrate_cmd.py - tools/chemenu/config.py - tools/chemenu/corpus_cache.py - tools/chemenu/prerequisites.py - tools/chemenu/search/ripgrep.py - tools/chemenu/tests/conftest.py - tools/chemenu/tests/test_dist_upgrade.py - tools/chemenu/tests/test_doctor.py - tools/chemenu/tests/test_preflight.py - tools/chemenu/toolpaths.py - tools/preflight.sh - tools/prerequisites.txt - tools/run_wikitool.py - tools/trace-hook - tools/wikitool --- .claude/settings.json | 2 +- .gitea/workflows/ci.yml | 29 +- .gitea/workflows/nightly.yml | 4 +- .gitea/workflows/release.yml | 4 +- .gitea/workflows/tracker-live.yml | 6 +- .github/hooks/wiki-trace.json | 44 +- .gitignore | 7 + .vibe/hooks.toml | 6 +- AGENTS.md | 4 + CHANGES.md | 30 +- EVALS.md | 14 +- INSTALL.md | 24 +- README.md | 5 +- VERSION | 2 +- instructions/bootstrap.md | 13 +- instructions/preflight.md | 90 ++++ instructions/setup-instance.md | 12 +- instructions/upgrade-instance.md | 15 +- tools/CONTRACT.md | 12 +- tools/README.md | 29 +- tools/chemenu/cli.py | 14 +- tools/chemenu/commands/dist_cmd.py | 6 +- tools/chemenu/commands/docs_verify.py | 4 +- tools/chemenu/commands/doctor.py | 104 ++++- tools/chemenu/commands/git_publish.py | 7 +- tools/chemenu/commands/migrate_cmd.py | 6 +- tools/chemenu/config.py | 4 +- tools/chemenu/corpus_cache.py | 4 +- tools/chemenu/prerequisites.py | 117 +++++ tools/chemenu/search/ripgrep.py | 4 +- tools/chemenu/tests/conftest.py | 10 +- tools/chemenu/tests/test_dist_upgrade.py | 4 +- tools/chemenu/tests/test_doctor.py | 84 +++- tools/chemenu/tests/test_preflight.py | 529 +++++++++++++++++++++++ tools/chemenu/toolpaths.py | 88 ++++ tools/preflight.sh | 520 ++++++++++++++++++++++ tools/prerequisites.txt | 22 + tools/run_wikitool.py | 14 + tools/trace-hook | 25 ++ tools/wikitool | 56 ++- 40 files changed, 1849 insertions(+), 125 deletions(-) create mode 100644 instructions/preflight.md create mode 100644 tools/chemenu/prerequisites.py create mode 100644 tools/chemenu/tests/test_preflight.py create mode 100644 tools/chemenu/toolpaths.py create mode 100755 tools/preflight.sh create mode 100644 tools/prerequisites.txt create mode 100644 tools/run_wikitool.py create mode 100755 tools/trace-hook diff --git a/.claude/settings.json b/.claude/settings.json index 98124b3..5b209ac 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -11,7 +11,7 @@ "hooks": [ { "type": "command", - "command": "./tools/trace_ingest.py --source claude-code --event prompt.submitted 2>/dev/null || true", + "command": "./tools/trace-hook --source claude-code --event prompt.submitted 2>/dev/null || true", "timeout": 5 } ] diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 4e0b57f..c5885b8 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -90,21 +90,27 @@ jobs: fetch-depth: 0 - name: Tool environment + # The preflight, not a venv block of our own: it is the one way a + # checkout gets set up (instructions/preflight.md), and tools/wikitool + # refuses to start until it has passed. Run twice - the second run must + # pass without changing anything, which is what an instance relies on + # when it re-runs it after every update. run: | set -eu git config --global --add safe.directory "$GITHUB_WORKSPACE" - python3 -m venv tools/.venv - tools/.venv/bin/pip install --quiet --upgrade pip - tools/.venv/bin/pip install --quiet -r tools/requirements.txt + tools/preflight.sh + cp .wikitool-tools.json /tmp/tools-first.json + tools/preflight.sh > /tmp/preflight-second.txt + cmp .wikitool-tools.json /tmp/tools-first.json # pytest-cov is CI-only: tools/requirements.txt describes what an # *instance* needs at runtime and ships with `dist export`, and an # instance does not measure this suite. Installed beside pytest for # the same reason pytest itself is. - tools/.venv/bin/pip install --quiet pytest pytest-cov + tools/.venv/bin/python -m pip install --quiet pytest pytest-cov # The MCP server's dependency is optional for an instance but not for # CI: its tests skip without it, and a skipped golden test is exactly # how the server's output and the CLI's would drift apart unnoticed. - tools/.venv/bin/pip install --quiet -r tools/requirements-mcp.txt + tools/.venv/bin/python -m pip install --quiet -r tools/requirements-mcp.txt - name: Tests # Not run with WIKI_TRACE=0: two telemetry tests assert that a trace is @@ -137,7 +143,7 @@ jobs: # $GITHUB_ENV, which only reaches the steps after the one that wrote it. run: | set -eu - tools/.venv/bin/pip install --quiet radicale + tools/.venv/bin/python -m pip install --quiet radicale .gitea/scripts/start-radicale.sh tools/.venv/bin/python /tmp/radicale - name: Live tracker suite (CalDAV) @@ -258,8 +264,15 @@ jobs: for template in kb/*/COLLECTION.md.template types/*.template; do cp "$template" "${template%.template}" done - python3 -m venv tools/.venv - tools/.venv/bin/pip install --quiet -r tools/requirements.txt + # A fresh instance refuses to run before its preflight (exit 42), so + # this proves both halves: the refusal, then the setup that ends it. + set +e + tools/wikitool doctor > /tmp/before-preflight.txt 2>&1 + refused=$? + set -e + test "$refused" -eq 42 + grep -q 'tools/preflight.sh' /tmp/before-preflight.txt + tools/preflight.sh tools/wikitool instructions sync tools/wikitool index rebuild tools/wikitool sources rebuild-index diff --git a/.gitea/workflows/nightly.yml b/.gitea/workflows/nightly.yml index 79f1703..0fe0a3b 100644 --- a/.gitea/workflows/nightly.yml +++ b/.gitea/workflows/nightly.yml @@ -79,9 +79,7 @@ jobs: git config --global --add safe.directory "$GITHUB_WORKSPACE" git config --global user.name "Nightly" git config --global user.email "nightly@example.invalid" - python3 -m venv tools/.venv - tools/.venv/bin/pip install --quiet --upgrade pip - tools/.venv/bin/pip install --quiet -r tools/requirements.txt + tools/preflight.sh tools/wikitool instructions sync - name: The instance is still correctly configured diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index a773310..6b9c825 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -54,9 +54,7 @@ jobs: run: | set -eu git config --global --add safe.directory "$GITHUB_WORKSPACE" - python3 -m venv tools/.venv - tools/.venv/bin/pip install --quiet --upgrade pip - tools/.venv/bin/pip install --quiet -r tools/requirements.txt + tools/preflight.sh - name: Resolve the version and refuse to re-release it id: version diff --git a/.gitea/workflows/tracker-live.yml b/.gitea/workflows/tracker-live.yml index 1ab8b7a..1c4a1f5 100644 --- a/.gitea/workflows/tracker-live.yml +++ b/.gitea/workflows/tracker-live.yml @@ -51,10 +51,8 @@ jobs: run: | set -eu git config --global --add safe.directory "$GITHUB_WORKSPACE" - python3 -m venv tools/.venv - tools/.venv/bin/pip install --quiet --upgrade pip - tools/.venv/bin/pip install --quiet -r tools/requirements.txt - tools/.venv/bin/pip install --quiet pytest radicale + tools/preflight.sh + tools/.venv/bin/python -m pip install --quiet pytest radicale - name: Start Radicale run: .gitea/scripts/start-radicale.sh tools/.venv/bin/python /tmp/radicale diff --git a/.github/hooks/wiki-trace.json b/.github/hooks/wiki-trace.json index 43e819d..c536393 100644 --- a/.github/hooks/wiki-trace.json +++ b/.github/hooks/wiki-trace.json @@ -4,8 +4,8 @@ "sessionStart": [ { "type": "command", - "bash": "./tools/trace_ingest.py --source copilot-cli --event session.start 2>/dev/null || true", - "powershell": "python tools/trace_ingest.py --source copilot-cli --event session.start 2>$null; exit 0", + "bash": "./tools/trace-hook --source copilot-cli --event session.start 2>/dev/null || true", + "powershell": ".\\tools\\.venv\\Scripts\\python.exe tools\\trace_ingest.py --source copilot-cli --event session.start 2>$null; exit 0", "cwd": ".", "timeoutSec": 5 } @@ -13,8 +13,8 @@ "sessionEnd": [ { "type": "command", - "bash": "./tools/trace_ingest.py --source copilot-cli --event session.end 2>/dev/null || true", - "powershell": "python tools/trace_ingest.py --source copilot-cli --event session.end 2>$null; exit 0", + "bash": "./tools/trace-hook --source copilot-cli --event session.end 2>/dev/null || true", + "powershell": ".\\tools\\.venv\\Scripts\\python.exe tools\\trace_ingest.py --source copilot-cli --event session.end 2>$null; exit 0", "cwd": ".", "timeoutSec": 5 } @@ -22,8 +22,8 @@ "userPromptSubmitted": [ { "type": "command", - "bash": "./tools/trace_ingest.py --source copilot-cli --event prompt.submitted 2>/dev/null || true", - "powershell": "python tools/trace_ingest.py --source copilot-cli --event prompt.submitted 2>$null; exit 0", + "bash": "./tools/trace-hook --source copilot-cli --event prompt.submitted 2>/dev/null || true", + "powershell": ".\\tools\\.venv\\Scripts\\python.exe tools\\trace_ingest.py --source copilot-cli --event prompt.submitted 2>$null; exit 0", "cwd": ".", "timeoutSec": 5 } @@ -31,8 +31,8 @@ "preToolUse": [ { "type": "command", - "bash": "./tools/trace_ingest.py --source copilot-cli --event tool.pre 2>/dev/null || true", - "powershell": "python tools/trace_ingest.py --source copilot-cli --event tool.pre 2>$null; exit 0", + "bash": "./tools/trace-hook --source copilot-cli --event tool.pre 2>/dev/null || true", + "powershell": ".\\tools\\.venv\\Scripts\\python.exe tools\\trace_ingest.py --source copilot-cli --event tool.pre 2>$null; exit 0", "cwd": ".", "timeoutSec": 5 } @@ -40,8 +40,8 @@ "postToolUse": [ { "type": "command", - "bash": "./tools/trace_ingest.py --source copilot-cli --event tool.post 2>/dev/null || true", - "powershell": "python tools/trace_ingest.py --source copilot-cli --event tool.post 2>$null; exit 0", + "bash": "./tools/trace-hook --source copilot-cli --event tool.post 2>/dev/null || true", + "powershell": ".\\tools\\.venv\\Scripts\\python.exe tools\\trace_ingest.py --source copilot-cli --event tool.post 2>$null; exit 0", "cwd": ".", "timeoutSec": 5 } @@ -49,8 +49,8 @@ "postToolUseFailure": [ { "type": "command", - "bash": "./tools/trace_ingest.py --source copilot-cli --event tool.error 2>/dev/null || true", - "powershell": "python tools/trace_ingest.py --source copilot-cli --event tool.error 2>$null; exit 0", + "bash": "./tools/trace-hook --source copilot-cli --event tool.error 2>/dev/null || true", + "powershell": ".\\tools\\.venv\\Scripts\\python.exe tools\\trace_ingest.py --source copilot-cli --event tool.error 2>$null; exit 0", "cwd": ".", "timeoutSec": 5 } @@ -58,8 +58,8 @@ "errorOccurred": [ { "type": "command", - "bash": "./tools/trace_ingest.py --source copilot-cli --event session.error 2>/dev/null || true", - "powershell": "python tools/trace_ingest.py --source copilot-cli --event session.error 2>$null; exit 0", + "bash": "./tools/trace-hook --source copilot-cli --event session.error 2>/dev/null || true", + "powershell": ".\\tools\\.venv\\Scripts\\python.exe tools\\trace_ingest.py --source copilot-cli --event session.error 2>$null; exit 0", "cwd": ".", "timeoutSec": 5 } @@ -67,8 +67,8 @@ "subagentStart": [ { "type": "command", - "bash": "./tools/trace_ingest.py --source copilot-cli --event subagent.start 2>/dev/null || true", - "powershell": "python tools/trace_ingest.py --source copilot-cli --event subagent.start 2>$null; exit 0", + "bash": "./tools/trace-hook --source copilot-cli --event subagent.start 2>/dev/null || true", + "powershell": ".\\tools\\.venv\\Scripts\\python.exe tools\\trace_ingest.py --source copilot-cli --event subagent.start 2>$null; exit 0", "cwd": ".", "timeoutSec": 5 } @@ -76,8 +76,8 @@ "subagentStop": [ { "type": "command", - "bash": "./tools/trace_ingest.py --source copilot-cli --event subagent.stop 2>/dev/null || true", - "powershell": "python tools/trace_ingest.py --source copilot-cli --event subagent.stop 2>$null; exit 0", + "bash": "./tools/trace-hook --source copilot-cli --event subagent.stop 2>/dev/null || true", + "powershell": ".\\tools\\.venv\\Scripts\\python.exe tools\\trace_ingest.py --source copilot-cli --event subagent.stop 2>$null; exit 0", "cwd": ".", "timeoutSec": 5 } @@ -85,8 +85,8 @@ "preCompact": [ { "type": "command", - "bash": "./tools/trace_ingest.py --source copilot-cli --event compaction 2>/dev/null || true", - "powershell": "python tools/trace_ingest.py --source copilot-cli --event compaction 2>$null; exit 0", + "bash": "./tools/trace-hook --source copilot-cli --event compaction 2>/dev/null || true", + "powershell": ".\\tools\\.venv\\Scripts\\python.exe tools\\trace_ingest.py --source copilot-cli --event compaction 2>$null; exit 0", "cwd": ".", "timeoutSec": 5 } @@ -94,8 +94,8 @@ "agentStop": [ { "type": "command", - "bash": "./tools/trace_ingest.py --source copilot-cli --event turn.end 2>/dev/null || true", - "powershell": "python tools/trace_ingest.py --source copilot-cli --event turn.end 2>$null; exit 0", + "bash": "./tools/trace-hook --source copilot-cli --event turn.end 2>/dev/null || true", + "powershell": ".\\tools\\.venv\\Scripts\\python.exe tools\\trace_ingest.py --source copilot-cli --event turn.end 2>$null; exit 0", "cwd": ".", "timeoutSec": 5 } diff --git a/.gitignore b/.gitignore index 1934eaf..d71ea5f 100644 --- a/.gitignore +++ b/.gitignore @@ -141,6 +141,13 @@ npm-debug.log* # configured; `doctor` reports which. /.wikitool-tasks.json +# Tool paths recorded by the preflight (tools/preflight.sh / .ps1, see +# instructions/preflight.md): the absolute paths of python, git, rg - and pwsh +# on Windows - as this machine has them. Per-checkout for the plainest reason of +# all: a path on one computer means nothing on another. Absent means the +# preflight has not run, and `tools/wikitool` refuses to start (exit 42). +/.wikitool-tools.json + # Live-suite tracker profiles (Gitea #156): one file per tracker of the user's own that the # live suite may be pointed at (`CHEMENU_LIVE_PROFILE=`, # instructions/dev/tracker-testing.md). They carry the same credentials as the file above diff --git a/.vibe/hooks.toml b/.vibe/hooks.toml index f6c8be5..29bd8fb 100644 --- a/.vibe/hooks.toml +++ b/.vibe/hooks.toml @@ -14,7 +14,7 @@ name = "wiki-trace-pre-tool" type = "pre_tool" description = "Record an intended tool call. Observational only - never decides." -command = "./tools/trace_ingest.py --source mistral-vibe --event tool.pre 2>/dev/null || true" +command = "./tools/trace-hook --source mistral-vibe --event tool.pre 2>/dev/null || true" timeout = 5.0 # `strict = false` is the default and is spelled out here because it is the # safety property that matters: under a non-strict hook, a crash or a timeout is @@ -26,7 +26,7 @@ strict = false name = "wiki-trace-post-tool" type = "post_tool" description = "Record the outcome of a tool call: status, output, duration." -command = "./tools/trace_ingest.py --source mistral-vibe --event tool.post 2>/dev/null || true" +command = "./tools/trace-hook --source mistral-vibe --event tool.post 2>/dev/null || true" timeout = 5.0 strict = false @@ -35,5 +35,5 @@ strict = false name = "wiki-trace-post-agent" type = "post_agent" description = "Record the end of an agent turn." -command = "./tools/trace_ingest.py --source mistral-vibe --event turn.end 2>/dev/null || true" +command = "./tools/trace-hook --source mistral-vibe --event turn.end 2>/dev/null || true" timeout = 5.0 diff --git a/AGENTS.md b/AGENTS.md index b2a1006..a653fc0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,10 @@ maintained permanently; anything mechanical is done by `tools/wikitool`, never b ## Bootstrap +**No `tools/wikitool` call works before the preflight has passed in this checkout** - it exits 42 +and names it: [instructions/preflight.md](instructions/preflight.md). On its own exit 42, show +the output verbatim and wait; never install or work around what it reports. + `.agents/skills/` and `.claude/skills/` are generated and **not committed**. If they are missing or empty - a fresh clone - the harness offers no skills until they are published: diff --git a/CHANGES.md b/CHANGES.md index 2b99fb7..ce26740 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -59,7 +59,7 @@ concern - readable here, never shipped as something to parse. --- -## 8.0.0-beta.13 - 2026-09-30 - raw/CONTRACT.md points at the path budget for a name accepted from incoming/ +## 8.0.0-beta.14 - 2026-09-30 - Preflight: prerequisites checked and tool paths recorded before wikitool runs (#151, POSIX half) **Author:** Torben Nehmer @@ -67,6 +67,7 @@ concern - readable here, never shipped as something to parse. - Page titles must form valid, unique file names on Windows and macOS: new and rename refuse forbidden characters, reserved names (including INDEX and COLLECTION), a trailing dot or space, and titles that collide with another page by case or Unicode normalization; lint reports existing violations as hard errors - rename each affected page with tools/wikitool rename - publish without --no-push now exits 1 before committing when the remote is unreachable or not configured, where it used to commit locally and fail at the push - an offline session or a local-only instance must pass --no-push - new, rename, move and raw accept refuse a target whose path below the instance root is over 160 characters (UTF-16 code units); lint reports existing files over it as Long Paths (advisory) - rename each affected page with tools/wikitool rename, and shorten an incoming/ file name before raw accept +- tools/wikitool now refuses to start (exit 42) until tools/preflight.sh has passed in the checkout - after updating, run tools/preflight.sh once: it checks Python, git and ripgrep, records their paths in .wikitool-tools.json and sets up tools/.venv **Migration:** none required - No page format changes; the rule only refuses titles, and each affected page is renamed individually with tools/wikitool rename @@ -75,6 +76,7 @@ concern - readable here, never shipped as something to parse. - wikitool: one data record per command - `-h`, index and CONTRACT.md render from cli_contract (Gitea #121 Phase 1) - dist upgrade --latest: one-command update from the release feed - Page titles must form valid, unique file names on Windows and macOS +- Preflight: prerequisites checked and tool paths recorded before wikitool runs (#151, POSIX half) **Medium impact** - CalDAV task-tracker provider (Nextcloud Tasks, iOS Reminders); review reports unknown-value findings instead of skipping them @@ -123,6 +125,32 @@ concern - readable here, never shipped as something to parse. - raw/CONTRACT.md points at the path budget for a name accepted from incoming/ +### Preflight: prerequisites checked and tool paths recorded before wikitool runs (#151, POSIX half) + +The Windows install that prompted this found Python missing, then `rg`, and the agent worked +around each gap instead of stopping. `tools/preflight.sh` is now the one way a checkout gets +set up. It checks the tools `tools/prerequisites.txt` lists (Python 3.11+, git, ripgrep, and +PowerShell 7 on Windows) and records each one's absolute path in `.wikitool-tools.json`, which +is gitignored and per checkout. It then creates `tools/.venv` from the recorded Python with +`-m venv` and `-m pip`. Anything missing, too old or unusable ends in exit 42 with a numbered +block for the user: what, why, the command that fixes it, what next. `--set =` +takes a path the user names. It is POSIX sh because it has to run before Python is known to +exist, under dash, bash and Git Bash. On Windows it tries `python`, `py -3`, `python3` in that +order, never runs a Microsoft Store alias, and records `sys.executable`. With long paths off it +refuses an install folder over 95 characters, the other half of the 160-character path budget. + +`tools/wikitool` is POSIX sh now as well. It stops with exit 42 until the preflight has written +a complete file, reads both venv layouts, and starts Python through `tools/run_wikitool.py` +instead of `PYTHONPATH`. Every `git` and `rg` the package starts goes through +`chemenu.toolpaths`, so it comes from the recorded path rather than the session's `PATH`. A +path that has gone is an `ERROR` line naming the preflight, not a traceback. `doctor` gains +`tool-paths` and `install-dir`. The harness hooks start `trace_ingest.py` through +`tools/trace-hook`, under the venv's Python, because the script's `python3` shebang is the +Store alias in Git Bash. The CI workflows, `bootstrap.md`, `setup-instance.md` and +`upgrade-instance.md` run the preflight instead of their own venv steps. The rules an agent +follows around it are in `instructions/preflight.md`. The PowerShell half and the download +mode for a first install follow in the same candidate. + ### raw/CONTRACT.md points at the path budget for a name accepted from incoming/ The path budget's close-out review found the raw stage contract silent on it, although `raw diff --git a/EVALS.md b/EVALS.md index e1575cd..5fd319b 100644 --- a/EVALS.md +++ b/EVALS.md @@ -128,7 +128,8 @@ Verified against vendor documentation on 2026-08-23. ### Claude Code -`.claude/settings.json` wires `UserPromptSubmit` to `tools/trace_ingest.py`. That is what makes +`.claude/settings.json` wires `UserPromptSubmit` to `tools/trace_ingest.py`, through +`tools/trace-hook` like every hook here (see the load-bearing details under Copilot CLI). That is what makes `clearance-ended-the-turn` scorable here: without a `prompt.submitted` event there is no turn boundary to place an exit-42 call and its `--confirm` on either side of, and the rule reports "cannot say" instead of a verdict. @@ -157,8 +158,17 @@ own decision-document schema verified against a live CLI first (this repo has no unverified, per the same rule that governed the Vibe adapter: an adapter that cannot be verified is not written. -Two details in that file are load-bearing: +Three details in that file are load-bearing, and the first holds for all three hook +configurations: +- **The interpreter is the venv's, never the script's shebang.** Each `bash` command is + `./tools/trace-hook ...`, which runs `trace_ingest.py` with `tools/.venv`'s Python in either + venv layout; each `powershell` command names `tools\.venv\Scripts\python.exe` directly. A + hook command is a fixed string and cannot read `.wikitool-tools.json`, but the venv is + created from the recorded interpreter at a fixed place, so it stands in for it. The shebang + (`python3`) was the Microsoft Store alias in Git Bash on Windows, which made every hook there + a silent no-op. Before the preflight has created the venv, `trace-hook` records nothing and + exits 0. - **Every command ends in `|| true`.** `preToolUse` hooks are *fail-closed*: a non-zero exit denies the tool call. Without the guard, a missing interpreter would turn the observer into a blocker that refuses every tool call in the session. (Timeouts are fail-open, so the diff --git a/INSTALL.md b/INSTALL.md index 83f0002..a28d860 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -18,6 +18,12 @@ Terminal auf dieser Maschine ist. - [ripgrep](https://github.com/BurntSushi/ripgrep) (`rg`) - wird von `search` und `sources coverage` gebraucht +Ob das alles da ist, prüft der Preflight (`tools/preflight.sh`), bevor irgendein +`wikitool`-Befehl läuft - der erste Schritt jeder Einrichtung, siehe +[instructions/preflight.md](instructions/preflight.md). Die maßgebliche Liste steht in +`tools/prerequisites.txt`. Fehlt etwas, hält der Agent an und zeigt eine Anleitung mit dem +Befehl, der es behebt; installieren muss man selbst, der Agent tut es nie. + ## Weg A: Release herunterladen Der kürzeste Weg zu einer eigenen Instanz - kein Checkout dieses Repos nötig. Jedes Release @@ -114,7 +120,7 @@ git clone https://gitea.nehmer.net/torben/chemenu.git cd chemenu ``` -Danach den Agenten `instructions/bootstrap.md` ausführen lassen (Werkzeugumgebung + Skills +Danach den Agenten `instructions/bootstrap.md` ausführen lassen (Preflight + Skills publizieren). Git-Repo, Autor-Identität und Inhalt existieren hier bereits. Ein Clone, der älter ist als die Personalization-Dateien, hat kein `USER.md`/`SOUL.md` - @@ -319,6 +325,15 @@ export WIKITOOL_UPDATE_TOKEN="" tools/wikitool version check ``` +**Tool-Pfade - schreibt der Preflight, nicht der Mensch.** `.wikitool-tools.json` im Repo-Root +hält die absoluten Pfade von Python, git und ripgrep, so wie der Preflight sie auf diesem Rechner +gefunden hat; `wikitool` startet git und rg von dort statt über `PATH`. Pro Checkout und +gitignored - ein Pfad auf einem Rechner sagt über den nächsten nichts. Fehlt die Datei oder ist +sie unvollständig, startet `tools/wikitool` nicht (Exit 42) und nennt den Preflight. Liegt ein +Tool woanders, als der Preflight sucht, nennt man den Pfad mit +`tools/preflight.sh --set rg=`; von Hand bearbeitet wird die Datei nicht. `doctor` meldet +unter `tool-paths`, ob alle Pfade noch stimmen. + **Aufgaben-Tracker anbinden - optional.** Der Wochenrückblick (`tools/wikitool review`, Skill `gtd-weekly-review`) gleicht die Projektseiten unter `kb/gtd/` gegen einen Aufgaben-Tracker ab. Welcher das ist, steht in `.wikitool-tasks.json` im Repo-Root - der dritten Datei dieser Art neben @@ -446,9 +461,10 @@ tools/wikitool instructions verify ## Troubleshooting -- **`wikitool: venv not found`** - Schritt "Werkzeugumgebung anlegen" aus - [instructions/bootstrap.md](instructions/bootstrap.md) bzw. - [instructions/setup-instance.md](instructions/setup-instance.md) wurde noch nicht ausgeführt. +- **`tools/wikitool` endet mit `STOP - this checkout is not set up yet` (Exit 42)** - der + Preflight ist in diesem Checkout noch nicht durchgelaufen, oder seit dem letzten Update nicht + mehr: `tools/preflight.sh` ausführen, siehe + [instructions/preflight.md](instructions/preflight.md). - **Der Agent bietet keine Skills an (`wiki-ingest`, `wiki-query`, ...)** - `.agents/skills/` und `.claude/skills/` sind generiert und nicht committet. `tools/wikitool instructions sync` ausführen, dann die Agent-Session neu starten (Harnesses lesen Skills nur beim Start). diff --git a/README.md b/README.md index 93d9c91..2e2c168 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,13 @@ Two starting points, depending on what you're doing - full walkthrough in [INSTA committed**. Publish them once: ```bash - cd tools && python3 -m venv .venv && .venv/bin/pip install -r requirements.txt && cd .. + tools/preflight.sh # checks python/git/rg, records their paths, creates tools/.venv tools/wikitool instructions sync ``` + `tools/wikitool` refuses to start (exit 42) until the preflight has passed; if it stops + instead, its output says what to install - `instructions/preflight.md`. + That copies each `instructions//SKILL.md` into `.agents/skills/` (GitHub Copilot, Codex CLI, Mistral Vibe) and `.claude/skills/` (Claude Code). Re-run it after changing a skill. Full procedure: `instructions/bootstrap.md`. diff --git a/VERSION b/VERSION index bb9ba9f..eb8b40a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -8.0.0-beta.13 +8.0.0-beta.14 diff --git a/instructions/bootstrap.md b/instructions/bootstrap.md index 5dc846b..13da528 100644 --- a/instructions/bootstrap.md +++ b/instructions/bootstrap.md @@ -1,7 +1,7 @@ --- type: types/instruction.md name: bootstrap -description: Prepare a fresh clone for work - create the tools venv and publish the skills into the harness directories, which are generated and not committed. +description: Prepare a fresh clone for work - run the preflight (tool paths and the tools venv) and publish the skills into the harness directories, which are generated and not committed. --- # Bootstrap a fresh clone @@ -19,15 +19,16 @@ they are published: the agent harness will not offer `wiki-ingest`, `wiki-query` ## Steps -1. **Create the tool environment** (once per clone): +1. **Run the preflight** (once per clone, and again after moving it) - see + [preflight.md](preflight.md). It records the tool paths in `.wikitool-tools.json` and + creates `tools/.venv`; until it exits 0, `tools/wikitool` refuses to start: ```bash - cd tools - python3 -m venv .venv - .venv/bin/pip install -r requirements.txt - cd .. + tools/preflight.sh ``` + On exit 42, show its output to the user verbatim and wait. + 2. **Publish the skills:** ```bash diff --git a/instructions/preflight.md b/instructions/preflight.md new file mode 100644 index 0000000..1896622 --- /dev/null +++ b/instructions/preflight.md @@ -0,0 +1,90 @@ +--- +type: types/instruction.md +name: preflight +description: Run the preflight before any wikitool command in a new, cloned, moved or updated checkout - it checks Python, git and ripgrep, records their paths in .wikitool-tools.json and sets up tools/.venv; on exit 42 show its output verbatim and wait for the user, never install or work around anything yourself. +--- +# Check the machine before anything else runs + +`tools/wikitool` does not start in a checkout the preflight has not passed in. It stops with +exit 42 and names this procedure instead - so there is no skipping it, only running it early +or being sent back to it. + +The preflight is a shell script, not a `wikitool` command, because it has to work before +Python is known to exist. It does three things, all inside the install folder: + +- checks the tools listed in `tools/prerequisites.txt` - Python 3.11 or newer, git, ripgrep + (`rg`) - and, on Windows, that the install folder is short enough for Windows' path limit; +- records the absolute path of each tool in `.wikitool-tools.json`, which `wikitool` then starts + them from instead of trusting whatever `PATH` a session inherited; +- creates `tools/.venv` from the recorded Python and installs `tools/requirements.txt` into it. + +## When to run + +- First step of every installation procedure: [setup-instance.md](setup-instance.md) and + [bootstrap.md](bootstrap.md) both start here. +- After every stack update ([upgrade-instance.md](upgrade-instance.md)) - a release can change + what the machine needs, or the requirements the venv holds. +- Whenever `tools/wikitool` exits 42 and names the preflight, and whenever `tools/wikitool doctor` + reports `tool-paths` or `install-dir` as `FAIL`. + +It is safe to run at any time: a second run on a ready checkout changes nothing and exits 0. + +## Steps + +1. **Run it** from the root of the checkout: + + ```bash + tools/preflight.sh + ``` + + This covers Linux, macOS and Git Bash on Windows, which is where Claude Code runs its + commands there. + +2. **Read the exit code.** + + | Exit | Meaning | What you do | + |---|---|---| + | 0 | Everything is in place | Continue with the procedure that sent you here | + | 42 | The user has to act | Step 3 | + | 1 | Called wrongly, or `tools/prerequisites.txt` is missing next to the script | Report the exact command and output to the user; do not retry blindly | + +3. **On exit 42, show the output to the user exactly as it is, then stop and wait.** It is + written for someone without an IT background: each numbered block says what is missing, why + it matters, the command that fixes it, and what happens next. When the user's language is not + the language of the output, add a translation below it - never instead of it, since the + commands inside have to reach them unchanged. + + While you wait, **install nothing, and work around nothing** - not with the user's consent + either. No package manager call, no other Python, no WSL, no hand-written + `.wikitool-tools.json`, no `wikitool` command "to see whether it works anyway". The command in + the output is for the user to run; how their machine is administered is theirs to decide. + +4. **When the user says it is done, run the preflight again.** Repeat steps 2-4 until it exits 0. + + When the user tells you where a tool is installed instead, pass the path on: + + ```bash + tools/preflight.sh --set rg=/opt/ripgrep/rg + ``` + + `--set =` may be given several times. A path that does not work is refused with + exit 42 and nothing is written; a working one is recorded and kept on later runs, even though + the tool is still not on `PATH`. + +## Decision points + +- **The output names a folder that is too long.** Only on Windows with long paths off: the + install folder may be at most 95 characters, because every file of the wiki below it has to + stay within 259. Moving the wiki to a shorter folder is the user's step; do not try to shorten + paths inside the wiki instead. +- **The venv or its libraries could not be installed.** The output carries the last lines of + what Python or pip said. A network, proxy or security-product cause is for the user - or + whoever administers their machine - to resolve; do not retry with other flags. +- **`.wikitool-tools.json` looks wrong.** Never edit it. Run the preflight again, with `--set` for + a path the user names; `doctor` reports whether the result holds. + +## Scope + +Not a wiki content procedure - it touches nothing under `kb/`, `raw/`, `work/` or `reports/`. +It does not configure identity, remotes or the harness either; those are later steps of +[setup-instance.md](setup-instance.md). diff --git a/instructions/setup-instance.md b/instructions/setup-instance.md index c8fa200..5ea4de8 100644 --- a/instructions/setup-instance.md +++ b/instructions/setup-instance.md @@ -193,15 +193,17 @@ and ready for its first ingest. `FAIL`, and so is one still carrying the sentinel - a renamed template is not a filled-in one. -7. **Create the tool environment** (details: [bootstrap.md](bootstrap.md)): +7. **Run the preflight** ([preflight.md](preflight.md)). It checks Python, git and ripgrep, + records their paths in `.wikitool-tools.json` and creates `tools/.venv` - no `tools/wikitool` + call works before it has passed: ```bash - cd tools - python3 -m venv .venv - .venv/bin/pip install -r requirements.txt - cd .. + tools/preflight.sh ``` + On exit 42, show its output to the user verbatim and wait; run it again once they have + acted. Continue here only after it exits 0. + 8. **Publish the skills:** ```bash diff --git a/instructions/upgrade-instance.md b/instructions/upgrade-instance.md index 36df18a..8d15fbb 100644 --- a/instructions/upgrade-instance.md +++ b/instructions/upgrade-instance.md @@ -150,8 +150,19 @@ fresh clone ([bootstrap.md](bootstrap.md)), and not for the clone-with-upstream It writes, and commits nothing. -8. **Republish the skills.** `tools/wikitool instructions sync` - the published skill directories - are copies, so until this runs the harness is still offering the previous release's skills. +8. **Run the preflight, then republish the skills.** The release may need other tools or + other libraries than the one it replaced, and `tools/wikitool` refuses to start (exit 42) + until the preflight has passed against the new `tools/` - an instance upgrading from a + release without one has never run it at all. On its exit 42, show the output verbatim and + wait ([preflight.md](preflight.md)): + + ```bash + tools/preflight.sh + tools/wikitool instructions sync + ``` + + `instructions sync` is needed because the published skill directories are copies: until it + runs, the harness is still offering the previous release's skills. 9. **Verify the machinery, and fix what the release said would need fixing:** diff --git a/tools/CONTRACT.md b/tools/CONTRACT.md index 3c16871..fccf320 100644 --- a/tools/CONTRACT.md +++ b/tools/CONTRACT.md @@ -54,14 +54,15 @@ file end to end is for changing the CLI itself. Full bootstrap for a fresh clone - including publishing the skills, which are not committed - is [`instructions/bootstrap.md`](../instructions/bootstrap.md). The -environment alone: +environment alone is the preflight ([`instructions/preflight.md`](../instructions/preflight.md)), +from the repo root: ```bash -cd tools -python3 -m venv .venv -.venv/bin/pip install -r requirements.txt +tools/preflight.sh ``` +Until it has passed, every `tools/wikitool` call exits 42 and names it. + ## Usage Run from the repo root: @@ -3220,6 +3221,8 @@ Check that this instance is correctly configured. **NOTES** - Checks dependencies (Python, ripgrep), author resolution, stack version, git identity/branch/remote, published skills, the kb/raw/reports/work/instructions structure, and generated files. +- Tool paths (`tool-paths`): `.wikitool-tools.json` written by a preflight that finished, every tool `tools/prerequisites.txt` names for this platform recorded, and every recorded path still there - a `FAIL` otherwise, fixed by running `tools/preflight.sh` again. +- Install folder (`install-dir`): on Windows with long paths off, a `FAIL` when the folder holding `tools/` is longer than `tools/prerequisites.txt` allows (95 characters) - the limit the preflight enforces before it sets anything up. - Personalization: `USER.md`/`SOUL.md` present **and** filled - a file still carrying the template's sentinel is a `FAIL`. - KB conventions: `kb/CONVENTIONS.md` present, unsentinelled, and naming all three tool-owned section headings - a `FAIL` on any of the three. - Environment note: `ENVIRONMENT.md` is optional, so absent is `OK`; a still-templated one is a `WARN`. @@ -3235,6 +3238,7 @@ Check that this instance is correctly configured. **SEE ALSO** - `instructions/setup-instance.md` - the setup steps most findings point back to +- `instructions/preflight.md` - what `tool-paths` and `install-dir` point back to - `INSTALL.md` § "Konfiguration" - the per-checkout configuration files - `EVALS.md` - telemetry state and caps - `instructions/session-setup.md` - setting `WIKITOOL_SESSION_ID` diff --git a/tools/README.md b/tools/README.md index 3ea0e92..d2c2493 100644 --- a/tools/README.md +++ b/tools/README.md @@ -18,26 +18,45 @@ file deliberately has none - and `docs verify` now enforces that. ## Setup ```bash -cd tools -python3 -m venv .venv -.venv/bin/pip install -r requirements.txt +tools/preflight.sh # from the repo root ``` +The preflight is the one way a checkout gets its environment: it checks the +tools listed in `prerequisites.txt`, records their absolute paths in +`../.wikitool-tools.json`, creates `.venv` and installs `requirements.txt` into +it with `-m pip`. It is POSIX sh because it has to run before Python is known +to exist; exit 42 means the user has to act, and its output says how. The +procedure an agent follows around it is `instructions/preflight.md`. + +`tools/wikitool` refuses to start (exit 42) until the preflight has written a +*complete* `.wikitool-tools.json` and the venv exists. Past that, every `git` +and `rg` the package starts goes through `chemenu/toolpaths.py`, which reads +the recorded path - or, with no file at all (the test suite, a bare +`python -m chemenu.cli`), falls back to the bare name. A file that is present +but names a path that has gone raises `ToolPathError`, which the CLI turns +into an `ERROR` line pointing at the preflight rather than a traceback. + `jsonschema` and `PyYAML` are hard dependencies, not optional extras: schema validation is the tool's whole safety net, so `cli.py` fails loudly with the -install command rather than degrading silently. +fix rather than degrading silently. ## Layout ``` tools/ - wikitool entry point + wikitool entry point (POSIX sh): stops with exit 42 until the preflight has passed + run_wikitool.py what the launcher runs with the venv's Python - puts chemenu on sys.path without PYTHONPATH + preflight.sh checks prerequisites.txt, records .wikitool-tools.json, creates .venv (POSIX sh) + prerequisites.txt what the machine needs, one `|`-separated line per tool - read by the preflight and `doctor` + trace-hook what the harness hooks call: trace_ingest.py under the venv's Python chemenu/ cli.py Typer app: registers every command, runs the budget gate, renders `-h`/`--help` from cli_contract cli_contract.py one data record per command (name, synopsis, properties, exit status) - the source `-h`, the index and CONTRACT.md's generated region render from config.py repo layout: root resolution and every path under it api.py the in-process entry point - point Chemenu at a corpus and read it errors.py ChemenuError / ValidationError / BackendError + toolpaths.py where git and rg are started from: .wikitool-tools.json, bare name only without the file + prerequisites.py prerequisites.txt read from Python, plus the platform and long-path questions `doctor` asks corpus_cache.py one parsed corpus per commit, never cached while the tree is dirty kb_scan.py page iteration/loading over kb/ blocks.py generated regions in a page body, found by marker rather than by heading diff --git a/tools/chemenu/cli.py b/tools/chemenu/cli.py index 33903e4..36189da 100644 --- a/tools/chemenu/cli.py +++ b/tools/chemenu/cli.py @@ -21,6 +21,7 @@ import typer.core as _typer_core _typer_core.HAS_RICH = False from chemenu import cli_contract # noqa: E402 - after the HAS_RICH patch, which must land first +from chemenu import toolpaths # noqa: E402 try: from chemenu.commands import ( @@ -59,8 +60,9 @@ except ModuleNotFoundError as exc: # degradation or a raw traceback. sys.stderr.write( f"wikitool: missing required dependency '{exc.name}'.\n" - "This is not optional - schema validation depends on it. Run:\n" - " cd tools && .venv/bin/pip install -r requirements.txt\n" + "This is not optional - schema validation depends on it. Run the preflight,\n" + "which (re)installs tools/.venv from tools/requirements.txt:\n" + " tools/preflight.sh\n" ) sys.exit(1) @@ -326,6 +328,14 @@ def _run_traced(command: str, args: list[str], charged: bool = False) -> None: code = exc.code exit_code = code if isinstance(code, int) else (0 if code is None else 1) raise + except toolpaths.ToolPathError as exc: + # Raised from wherever git or rg is about to start, often deep inside a + # helper that treats a missing tool as "no answer". It is neither a + # crash nor a validation error to retry: the fix is the preflight, so + # it gets the ERROR line and exit 1 rather than a traceback. + exit_code = 1 + print(f"ERROR {exc}", file=sys.stdout) + raise SystemExit(1) from None except BaseException: exit_code = 1 raise diff --git a/tools/chemenu/commands/dist_cmd.py b/tools/chemenu/commands/dist_cmd.py index 8d0cbe5..14cb56a 100644 --- a/tools/chemenu/commands/dist_cmd.py +++ b/tools/chemenu/commands/dist_cmd.py @@ -54,6 +54,7 @@ from chemenu import ( kb_state, ownership, toc, + toolpaths, version as version_mod, ) from chemenu.commands._util import console, fail, rel_path, success, today_iso @@ -953,7 +954,7 @@ def _git_working_tree_status() -> Optional[str]: repository at all - which is a valid, if unprotected, state for a tarball instance, not a reason to refuse.""" result = subprocess.run( - ["git", "-C", str(config.ROOT), "status", "--porcelain"], + [toolpaths.git(), "-C", str(config.ROOT), "status", "--porcelain"], capture_output=True, text=True, ) @@ -1515,6 +1516,7 @@ def run_upgrade( summary += ( " Nothing was committed and nothing is verified yet." " `instructions/upgrade-instance.md` carries the order for everything that follows" - " and resumes at `wikitool instructions sync`." + " and resumes with the preflight (`tools/preflight.sh`), which `tools/wikitool` now" + " refuses to run without." ) success(summary) diff --git a/tools/chemenu/commands/docs_verify.py b/tools/chemenu/commands/docs_verify.py index 3fadc0a..93eb5ff 100644 --- a/tools/chemenu/commands/docs_verify.py +++ b/tools/chemenu/commands/docs_verify.py @@ -68,7 +68,7 @@ from typing import Optional import typer -from chemenu import blocks, cli_contract, config, conventions, kb_collections, markdown_code, toc, version as version_mod +from chemenu import blocks, cli_contract, config, conventions, kb_collections, markdown_code, toc, toolpaths, version as version_mod from chemenu.commands import dist_cmd from chemenu.commands._util import fail, rel_path, success @@ -842,7 +842,7 @@ def _git(args: list[str], stdin: Optional[str] = None) -> Optional[subprocess.Co are unknowable rather than wrong.""" try: return subprocess.run( - ["git", *args], cwd=config.ROOT, capture_output=True, text=True, input=stdin + [toolpaths.git(), *args], cwd=config.ROOT, capture_output=True, text=True, input=stdin ) except OSError: return None diff --git a/tools/chemenu/commands/doctor.py b/tools/chemenu/commands/doctor.py index fc57753..e239762 100644 --- a/tools/chemenu/commands/doctor.py +++ b/tools/chemenu/commands/doctor.py @@ -11,6 +11,7 @@ remote yet, or no `WIKITOOL_SESSION_ID` set, is a valid state, not a fault. from __future__ import annotations import json as _json +import os import shutil import subprocess import sys @@ -20,7 +21,7 @@ from typing import Optional import typer from rich.console import Console -from chemenu import cli_contract, config, conventions, kb_collections, version as version_mod +from chemenu import cli_contract, config, conventions, kb_collections, prerequisites, toolpaths, version as version_mod from chemenu.commands import git_publish, instructions_cmd from chemenu.commands._util import rel_path from chemenu.session import ENV_VAR as SESSION_ENV_VAR @@ -40,33 +41,100 @@ class Check: def _git(args: list[str]) -> Optional[subprocess.CompletedProcess]: try: return subprocess.run( - ["git", *args], cwd=config.ROOT, capture_output=True, text=True, timeout=5 + [toolpaths.git(), *args], cwd=config.ROOT, capture_output=True, text=True, timeout=5 ) - except (OSError, subprocess.SubprocessError): + except (OSError, subprocess.SubprocessError, toolpaths.ToolPathError): + # A broken tool-paths file is reported once, by `check_tool_paths` - + # not as a crash from every check that happens to need git. return None +PREFLIGHT_FIX = "Run tools/preflight.sh" + + def check_python() -> Check: version = sys.version_info if version < (3, 11): return Check( "python", "FAIL", f"Python {version.major}.{version.minor} found, need >= 3.11", - "Install Python 3.11+ and recreate tools/.venv", + f"Install Python 3.11+, then: {PREFLIGHT_FIX}", ) return Check("python", "OK", f"Python {version.major}.{version.minor}.{version.micro}") def check_ripgrep() -> Check: - if shutil.which("rg"): - return Check("ripgrep", "OK", "rg found on PATH") + try: + rg = toolpaths.rg() + except toolpaths.ToolPathError as exc: + return Check("ripgrep", "FAIL", str(exc), PREFLIGHT_FIX) + if shutil.which(rg): + where = "on PATH" if rg == "rg" else f"at {rg}" + return Check("ripgrep", "OK", f"rg found {where}") return Check( - "ripgrep", "FAIL", "rg not found on PATH - `search` and `sources coverage` need it", - "Install ripgrep (e.g. `apt install ripgrep` / `brew install ripgrep`)", + "ripgrep", "FAIL", "rg not found - `search` and `sources coverage` need it", + f"Install ripgrep (e.g. `apt install ripgrep` / `brew install ripgrep`), then: {PREFLIGHT_FIX}", + ) + + +def check_tool_paths() -> Check: + """`.wikitool-tools.json`: written by a preflight that finished, and every + path in it still there. The launcher refuses to start without a complete + file, so a FAIL here is mostly a path that went away since - an uninstalled + or moved tool.""" + name = toolpaths.FILE_NAME + try: + data = toolpaths.load() + except toolpaths.ToolPathError as exc: + return Check("tool-paths", "FAIL", str(exc), PREFLIGHT_FIX) + if data is None: + return Check( + "tool-paths", "FAIL", f"{name} is missing - the preflight has not run in this checkout", + PREFLIGHT_FIX, + ) + if data.get("complete") is not True: + return Check( + "tool-paths", "FAIL", f"{name} is incomplete - the last preflight run stopped before the end", + PREFLIGHT_FIX, + ) + recorded = data["tools"] + needed = [tool.name for tool in prerequisites.load_manifest().tools_for(prerequisites.platform())] + unrecorded = [tool for tool in needed if not recorded.get(tool)] + gone = [f"{tool} ({path})" for tool, path in sorted(recorded.items()) + if not (isinstance(path, str) and os.path.isfile(path))] + problems = [] + if unrecorded: + problems.append("not recorded: " + ", ".join(unrecorded)) + if gone: + problems.append("no longer there: " + ", ".join(gone)) + if problems: + return Check("tool-paths", "FAIL", f"{name}: " + "; ".join(problems), PREFLIGHT_FIX) + return Check("tool-paths", "OK", f"{name}: " + ", ".join(sorted(recorded)) + " recorded and present") + + +def check_install_dir() -> Check: + """The install folder against Windows' MAX_PATH, the same limit the + preflight enforces before it unpacks anything.""" + problem = prerequisites.install_dir_problem() + if problem is None: + if prerequisites.platform() != "windows": + detail = "no folder length limit on this platform" + elif prerequisites.long_paths_enabled(): + detail = "Windows long paths are on - no folder length limit" + else: + detail = "fits Windows' path limit" + return Check("install-dir", "OK", detail) + return Check( + "install-dir", "FAIL", problem[:1].upper() + problem[1:], + "Move the wiki to a shorter folder (for example C:\\Chemenu) and run the preflight there, " + "or have someone with administrator rights turn on long paths in Windows", ) def check_author() -> Check: - author = config.default_author() + try: + author = config.default_author() + except toolpaths.ToolPathError as exc: + return Check("author", "FAIL", f"`git config user.name` cannot be asked: {exc}", PREFLIGHT_FIX) if author is None: return Check( "author", "FAIL", "Neither $WIKI_AUTHOR nor `git config user.name` resolves", @@ -313,10 +381,8 @@ def check_publish_remotes() -> Check: f"Gate armed: {len(urls)} allowed push target(s) in " f"{config.PUBLISH_REMOTES_FILENAME}", ) - result = subprocess.run( - ["git", "remote"], cwd=config.ROOT, capture_output=True, text=True - ) - remotes = [r for r in result.stdout.split() if r] + result = _git(["remote"]) + remotes = [r for r in result.stdout.split() if r] if result is not None else [] if len(remotes) > 1: return Check( "publish-remotes", "WARN", @@ -600,7 +666,9 @@ def check_kb_version() -> Check: def run_doctor() -> list[Check]: checks: list[Check] = [ check_python(), + check_tool_paths(), check_ripgrep(), + check_install_dir(), check_author(), check_stack_version(), check_kb_version(), @@ -635,6 +703,13 @@ def run_doctor() -> list[Check]: "Checks dependencies (Python, ripgrep), author resolution, stack version, git " "identity/branch/remote, published skills, the kb/raw/reports/work/instructions " "structure, and generated files.", + "Tool paths (`tool-paths`): `.wikitool-tools.json` written by a preflight that " + "finished, every tool `tools/prerequisites.txt` names for this platform recorded, and " + "every recorded path still there - a `FAIL` otherwise, fixed by running " + "`tools/preflight.sh` again.", + "Install folder (`install-dir`): on Windows with long paths off, a `FAIL` when the " + "folder holding `tools/` is longer than `tools/prerequisites.txt` allows (95 " + "characters) - the limit the preflight enforces before it sets anything up.", "Personalization: `USER.md`/`SOUL.md` present **and** filled - a file still carrying " "the template's sentinel is a `FAIL`.", "KB conventions: `kb/CONVENTIONS.md` present, unsentinelled, and naming all three " @@ -674,6 +749,7 @@ def run_doctor() -> list[Check]: ), see_also=( "`instructions/setup-instance.md` - the setup steps most findings point back to", + "`instructions/preflight.md` - what `tool-paths` and `install-dir` point back to", "`INSTALL.md` § \"Konfiguration\" - the per-checkout configuration files", "`EVALS.md` - telemetry state and caps", "`instructions/session-setup.md` - setting `WIKITOOL_SESSION_ID`", @@ -684,7 +760,7 @@ def doctor_command( ): """Check that this instance is correctly configured. \f - Dependencies, author, git identity/remote, published skills, structure, + Dependencies, recorded tool paths, install folder length, author, git identity/remote, published skills, structure, personalization, KB conventions, generated files, session scoping, telemetry state, whether the MCP `submit` tool is armed, and which task-tracker provider (if any) is configured for the GTD review. diff --git a/tools/chemenu/commands/git_publish.py b/tools/chemenu/commands/git_publish.py index 757af44..a3ab87b 100644 --- a/tools/chemenu/commands/git_publish.py +++ b/tools/chemenu/commands/git_publish.py @@ -60,7 +60,7 @@ from typing import NamedTuple, Optional import typer -from chemenu import cli_contract, config +from chemenu import cli_contract, config, toolpaths from chemenu.commands._util import fail, needs_clearance, success from chemenu.telemetry import emit @@ -81,6 +81,9 @@ GATE_EXEMPT_PREFIXES = ("work/",) def _run(args: list[str]) -> subprocess.CompletedProcess: + """Run a `git ...` argument list, starting git from its recorded path.""" + if args and args[0] == "git": + args = [toolpaths.git(), *args[1:]] return subprocess.run(args, cwd=config.ROOT, capture_output=True, text=True) @@ -266,7 +269,7 @@ def collect_changes(paths: list[str]) -> list[FileChange]: def run(args: list[str]) -> str: result = subprocess.run( - ["git", *args], cwd=config.ROOT, capture_output=True, text=True, env=env, + [toolpaths.git(), *args], cwd=config.ROOT, capture_output=True, text=True, env=env, ) if result.returncode != 0: fail(f"git {args[0]} failed:\n{result.stderr}") diff --git a/tools/chemenu/commands/migrate_cmd.py b/tools/chemenu/commands/migrate_cmd.py index d4fd951..0551064 100644 --- a/tools/chemenu/commands/migrate_cmd.py +++ b/tools/chemenu/commands/migrate_cmd.py @@ -21,7 +21,7 @@ from typing import Optional import typer -from chemenu import cli_contract, config, corpus_diff, kb_scan, kb_state, version as version_mod +from chemenu import cli_contract, config, corpus_diff, kb_scan, kb_state, toolpaths, version as version_mod from chemenu.commands._util import console, fail, rel_path, success, today_iso from chemenu.frontmatter_io import read_page from chemenu.page import Page @@ -500,7 +500,7 @@ def baseline_command( def _git_show(rev: str, relative: str) -> Optional[str]: result = subprocess.run( - ["git", "show", f"{rev}:{relative}"], + [toolpaths.git(), "show", f"{rev}:{relative}"], cwd=config.ROOT, capture_output=True, text=True, @@ -510,7 +510,7 @@ def _git_show(rev: str, relative: str) -> Optional[str]: def _paths_at(rev: str) -> Optional[list[str]]: result = subprocess.run( - ["git", "ls-tree", "-r", "--name-only", "-z", rev, "--", "kb"], + [toolpaths.git(), "ls-tree", "-r", "--name-only", "-z", rev, "--", "kb"], cwd=config.ROOT, capture_output=True, text=True, diff --git a/tools/chemenu/config.py b/tools/chemenu/config.py index c0a4f54..3460afe 100644 --- a/tools/chemenu/config.py +++ b/tools/chemenu/config.py @@ -299,9 +299,11 @@ def default_author() -> str | None: override = os.environ.get("WIKI_AUTHOR", "").strip() if override: return override + from chemenu import toolpaths # local: toolpaths imports this module + try: result = subprocess.run( - ["git", "config", "user.name"], + [toolpaths.git(), "config", "user.name"], cwd=_root(), capture_output=True, text=True, diff --git a/tools/chemenu/corpus_cache.py b/tools/chemenu/corpus_cache.py index 148fa7c..fc93127 100644 --- a/tools/chemenu/corpus_cache.py +++ b/tools/chemenu/corpus_cache.py @@ -28,7 +28,7 @@ import subprocess from pathlib import Path from typing import Optional -from chemenu import config +from chemenu import config, toolpaths from chemenu.page import Page @@ -62,7 +62,7 @@ def is_dirty(root: Optional[Path] = None, path: Optional[Path] = None) -> bool: def _git(args: list[str], root: Optional[Path] = None): try: return subprocess.run( - ["git", *args], + [toolpaths.git(), *args], cwd=root or config.ROOT, capture_output=True, text=True, diff --git a/tools/chemenu/prerequisites.py b/tools/chemenu/prerequisites.py new file mode 100644 index 0000000..853de7c --- /dev/null +++ b/tools/chemenu/prerequisites.py @@ -0,0 +1,117 @@ +"""`tools/prerequisites.txt` read from Python, for `wikitool doctor`. + +The file is the one list of what the machine needs; `tools/preflight.sh` and +`tools/preflight.ps1` read it too, which is why it is a `|`-separated line format +rather than anything a shell would need a parser for. This module answers the +same questions the preflight asks, so `doctor` can report afterwards what the +preflight enforced up front - a tool whose recorded path has gone, a checkout +moved into a folder too long for Windows. + +`CHEMENU_PREFLIGHT_PLATFORM` and `CHEMENU_PREFLIGHT_LONGPATHS` stand in for the +real platform and registry, exactly as they do for the preflight scripts; the +test suite is their only user. +""" +from __future__ import annotations + +import os +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from chemenu import config, titles + +ENV_PLATFORM = "CHEMENU_PREFLIGHT_PLATFORM" +ENV_LONGPATHS = "CHEMENU_PREFLIGHT_LONGPATHS" + + +@dataclass(frozen=True) +class Tool: + name: str + minimum: Optional[str] + platforms: str + label: str + why: str + + +@dataclass(frozen=True) +class Manifest: + limits: dict[str, int] + tools: tuple[Tool, ...] + + def tools_for(self, platform: str) -> tuple[Tool, ...]: + return tuple(t for t in self.tools if t.platforms in ("all", platform)) + + +def manifest_path() -> Path: + return config._PACKAGE_ROOT / "tools" / "prerequisites.txt" + + +def load_manifest(path: Optional[Path] = None) -> Manifest: + path = manifest_path() if path is None else path + limits: dict[str, int] = {} + tools: list[Tool] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip() or line.startswith("#"): + continue + fields = line.split("|") + if fields[0] == "limit": + limits[fields[1]] = int(fields[2]) + elif fields[0] == "tool": + minimum = None if fields[2] == "-" else fields[2] + tools.append(Tool(fields[1], minimum, fields[3], fields[4], fields[5])) + return Manifest(limits, tuple(tools)) + + +def platform() -> str: + """windows, macos or linux - the same three names the preflight uses.""" + forced = os.environ.get(ENV_PLATFORM, "").strip() + if forced: + return forced + if sys.platform == "win32": + return "windows" + if sys.platform == "darwin": + return "macos" + return "linux" + + +def long_paths_enabled() -> bool: + """Whether Windows lifts MAX_PATH on this machine. An unreadable key counts + as off, as it does in the preflight: the limit then protects a machine that + did not need it, rather than the other way round.""" + forced = os.environ.get(ENV_LONGPATHS, "").strip() + if forced: + return forced == "1" + if sys.platform != "win32": + return False + try: # pragma: no cover - Windows only + import winreg + + with winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, r"SYSTEM\CurrentControlSet\Control\FileSystem" + ) as key: + value, _ = winreg.QueryValueEx(key, "LongPathsEnabled") + return value == 1 + except OSError: # pragma: no cover - Windows only + return False + + +def install_dir() -> str: + """The folder that holds `tools/`, as the operating system spells it.""" + return str(config._PACKAGE_ROOT) + + +def install_dir_problem(folder: Optional[str] = None) -> Optional[str]: + """Why this install folder is too long for Windows with long paths off, or + None when it fits (or the question does not arise on this platform).""" + if platform() != "windows" or long_paths_enabled(): + return None + folder = install_dir() if folder is None else folder + limit = load_manifest().limits["install_dir_max"] + length = titles.path_length(folder) + if length <= limit: + return None + return ( + f"the install folder is {length} characters long and Windows allows at most " + f"{limit} here (long paths are off): {folder}" + ) diff --git a/tools/chemenu/search/ripgrep.py b/tools/chemenu/search/ripgrep.py index d166836..b87b944 100644 --- a/tools/chemenu/search/ripgrep.py +++ b/tools/chemenu/search/ripgrep.py @@ -23,7 +23,7 @@ import subprocess from pathlib import Path from typing import Iterable -from chemenu import config +from chemenu import config, toolpaths from chemenu.errors import BackendError from chemenu.page import Page from chemenu.search.base import page_key @@ -65,7 +65,7 @@ class RipgrepFailed(BackendError): def build_argv(query: SearchQuery, root: Path) -> list[str]: """The exact command line. Split out so a test can assert the safety properties above without running anything.""" - argv = ["rg", "--json", "--smart-case", "--glob", "*.md"] + argv = [toolpaths.rg(), "--json", "--smart-case", "--glob", "*.md"] if not query.regex: argv.append("--fixed-strings") # `--` terminates option parsing: a query starting with `-` is a search diff --git a/tools/chemenu/tests/conftest.py b/tools/chemenu/tests/conftest.py index 6937a47..bac0aab 100644 --- a/tools/chemenu/tests/conftest.py +++ b/tools/chemenu/tests/conftest.py @@ -4,7 +4,7 @@ from pathlib import Path import pytest -from chemenu import config, conventions +from chemenu import config, conventions, toolpaths from chemenu.frontmatter_io import write_page from chemenu.session import HARNESS_ENV_VARS from chemenu.telemetry import policy as telemetry_policy @@ -33,6 +33,8 @@ _WIKITOOL_ENV = ( "WIKITOOL_UPDATE_TOKEN", "CHEMENU_ROOT", "WIKITOOL_TASKS_CONFIG", + "CHEMENU_PREFLIGHT_PLATFORM", + "CHEMENU_PREFLIGHT_LONGPATHS", ) + tuple(var for var, _harness in HARNESS_ENV_VARS) # Environment git reads for identity or for where its repo lives. A stray @@ -156,6 +158,12 @@ def hermetic_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): for name in (*_WIKITOOL_ENV, *_GIT_ENV): monkeypatch.delenv(name, raising=False) + # `.wikitool-tools.json` beside this checkout records where the developer's + # git and rg live. A test must not depend on whether the preflight has run + # here, so every test sees no file - the bare-name fallback - unless it + # writes one of its own and points `toolpaths.tools_file` at it. + monkeypatch.setattr(toolpaths, "tools_file", lambda: home / toolpaths.FILE_NAME) + # The same hole as the environment above, one layer in: `config` resolves # its paths on access, and `monkeypatch.setattr(config, "KB_DIR", ...)` # undoes itself by writing the *resolved* old path back as a real diff --git a/tools/chemenu/tests/test_dist_upgrade.py b/tools/chemenu/tests/test_dist_upgrade.py index e2e16f8..6ab6474 100644 --- a/tools/chemenu/tests/test_dist_upgrade.py +++ b/tools/chemenu/tests/test_dist_upgrade.py @@ -345,7 +345,9 @@ def test_closing_report_points_at_the_upgrade_instruction(instance, tmp_path, ca out = " ".join(capsys.readouterr().out.split()) # rich wraps; rejoin first assert "instructions/upgrade-instance.md" in out - assert "instructions sync" in out + # The first thing the new launcher needs (Gitea #151): it refuses to start + # until the preflight has passed against the swapped-in tools/. + assert "tools/preflight.sh" in out @pytest.mark.parametrize("preserved", [".wikitool-kb.json", "CHANGES.md", "kb/log.md", "raw/notes/.gitkeep"]) diff --git a/tools/chemenu/tests/test_doctor.py b/tools/chemenu/tests/test_doctor.py index 07faff5..a9e528c 100644 --- a/tools/chemenu/tests/test_doctor.py +++ b/tools/chemenu/tests/test_doctor.py @@ -4,12 +4,14 @@ is missing.""" from __future__ import annotations import json +import shutil import subprocess +import sys from pathlib import Path import pytest -from chemenu import config, conventions +from chemenu import config, conventions, prerequisites, toolpaths from chemenu.commands import doctor, instructions_cmd @@ -17,6 +19,18 @@ def _git(root: Path, *args: str) -> None: subprocess.run(["git", *args], cwd=root, check=True, capture_output=True) +def _write_tools_file(path: Path, tools: dict, complete: bool = True) -> Path: + path.write_text( + json.dumps({"schema": 1, "complete": complete, "tools": tools}, indent=2) + "\n", + encoding="utf-8", + ) + return path + + +def _real_tools() -> dict: + return {"python": sys.executable, "git": shutil.which("git"), "rg": shutil.which("rg")} + + @pytest.fixture def instance(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: """A minimal, fully-configured wiki instance: a git repo with identity, @@ -72,6 +86,11 @@ def instance(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: _git(root, "config", "user.name", "Fixture Author") _git(root, "config", "user.email", "fixture@example.com") + # What the preflight would have written: every tool this platform needs, + # at the path it really has on this machine. + tools_file = _write_tools_file(root / toolpaths.FILE_NAME, _real_tools()) + monkeypatch.setattr(toolpaths, "tools_file", lambda: tools_file) + instructions_cmd.sync(force=False) return root @@ -558,3 +577,66 @@ def test_tasks_provider_names_an_active_override(instance, monkeypatch): checks = doctor.run_doctor() assert _status(checks, "tasks-provider") == "OK" assert "WIKITOOL_TASKS_CONFIG" in _detail(checks, "tasks-provider") + + +# --- tool paths and the install folder (Gitea #151) -------------------------- + + +def test_tool_paths_missing_file_is_a_fail_naming_the_preflight(instance): + (instance / toolpaths.FILE_NAME).unlink() + checks = doctor.run_doctor() + assert _status(checks, "tool-paths") == "FAIL" + fix = next(c.fix for c in checks if c.name == "tool-paths") + assert "tools/preflight.sh" in fix + + +def test_tool_paths_incomplete_run_is_a_fail(instance): + _write_tools_file(instance / toolpaths.FILE_NAME, _real_tools(), complete=False) + assert _status(doctor.run_doctor(), "tool-paths") == "FAIL" + + +def test_tool_paths_vanished_path_is_a_fail(instance): + tools = {**_real_tools(), "rg": str(instance / "gone" / "rg")} + _write_tools_file(instance / toolpaths.FILE_NAME, tools) + checks = doctor.run_doctor() + assert _status(checks, "tool-paths") == "FAIL" + assert "no longer there: rg" in _detail(checks, "tool-paths") + # The same broken path makes the ripgrep check fail instead of finding rg on PATH. + assert _status(checks, "ripgrep") == "FAIL" + + +def test_tool_paths_unrecorded_tool_is_a_fail(instance): + tools = _real_tools() + del tools["git"] + _write_tools_file(instance / toolpaths.FILE_NAME, tools) + checks = doctor.run_doctor() + assert _status(checks, "tool-paths") == "FAIL" + assert "not recorded: git" in _detail(checks, "tool-paths") + + +def test_tool_paths_complete_file_is_ok(instance): + assert _status(doctor.run_doctor(), "tool-paths") == "OK" + + +def _folder_of(length: int) -> str: + return "C:\\" + "x" * (length - 3) + + +@pytest.mark.parametrize("length, longpaths, expected", [ + (95, "0", "OK"), + (96, "0", "FAIL"), + (96, "1", "OK"), +]) +def test_install_dir_limit_at_the_boundary(instance, monkeypatch, length, longpaths, expected): + monkeypatch.setenv(prerequisites.ENV_PLATFORM, "windows") + monkeypatch.setenv(prerequisites.ENV_LONGPATHS, longpaths) + folder = _folder_of(length) + assert len(folder) == length + monkeypatch.setattr(prerequisites, "install_dir", lambda: folder) + assert _status(doctor.run_doctor(), "install-dir") == expected + + +def test_install_dir_is_not_limited_off_windows(instance, monkeypatch): + monkeypatch.setenv(prerequisites.ENV_PLATFORM, "linux") + monkeypatch.setattr(prerequisites, "install_dir", lambda: "/" + "x" * 300) + assert _status(doctor.run_doctor(), "install-dir") == "OK" diff --git a/tools/chemenu/tests/test_preflight.py b/tools/chemenu/tests/test_preflight.py new file mode 100644 index 0000000..fb7cd58 --- /dev/null +++ b/tools/chemenu/tests/test_preflight.py @@ -0,0 +1,529 @@ +"""tools/preflight.sh, the launcher, `tools/trace-hook` and `toolpaths` (Gitea #151). + +The preflight runs before Python is known to exist, so it is tested as what it +is: a POSIX shell script, run under every shell this machine has of dash and +bash, against a `PATH` built for the test. That `PATH` holds two directories - +the handful of utilities the script uses, symlinked one by one, and stubs for +the tools under test - so whether `rg` or `python3` is "installed" is decided +here and nowhere else. The stub Python fakes `-m venv` and `-m pip` as well, so +no test creates a real venv or touches the network; the CI workflows run the +preflight for real. +""" +from __future__ import annotations + +import json +import os +import shutil +import stat +import subprocess +import sys +from pathlib import Path + +import pytest + +from chemenu import config, prerequisites, titles, toolpaths + +TOOLS = config._PACKAGE_ROOT / "tools" + +# What preflight.sh, the launcher and trace-hook call besides shell built-ins +# and the tools under test. `iconv` is optional in the script itself. +UTILITIES = ("dirname", "uname", "sed", "tr", "wc", "iconv", "tail", "cat", "mv", "head", + "cut", "grep", "mkdir", "cp", "rm") + + +def _shells() -> list[str]: + found = {} + for name in ("dash", "bash", "sh"): + path = shutil.which(name) + if path: + found.setdefault(os.path.realpath(path), path) + return sorted(found.values()) + + +SHELLS = _shells() + + +def _executable(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + return path + + +FAKE_PYTHON = """#!/bin/sh +# A Python that answers the preflight's probe as itself, and fakes venv and pip. +case "$1" in + -c) + case "$2" in + *hashlib*) exec "{real}" "$@" ;; + esac + printf '%s\\n%s\\n' "${{FAKE_PY_VERSION:-3.13}}" "$0" + exit 0 ;; + -m) + case "$2" in + venv) + shift 2 + [ "$1" = --clear ] && shift + if [ -n "${{FAKE_VENV_FAIL:-}}" ]; then + echo "Error: ensurepip is not available" >&2 + exit 1 + fi + mkdir -p "$1/bin" + cp "$0" "$1/bin/python" + exit 0 ;; + pip) + shift 2 + [ "$1" = --version ] && exit 0 + echo "$*" >> "$PIP_LOG" + if [ -n "${{FAKE_PIP_FAIL:-}}" ]; then + echo "ERROR: Could not find a version that satisfies the requirement (network unreachable)" >&2 + exit 1 + fi + exit 0 ;; + esac ;; +esac +exit 1 +""" + + +class Machine: + """A stack tree plus a PATH of stubs, for one preflight run or several.""" + + def __init__(self, base: Path, root_name: str = "instance"): + self.base = base + self.root = base / root_name + self.tools = self.root / "tools" + self.tools.mkdir(parents=True) + for name in ("preflight.sh", "prerequisites.txt", "wikitool", "run_wikitool.py", "trace-hook"): + shutil.copy2(TOOLS / name, self.tools / name) + (self.tools / "requirements.txt").write_text("PyYAML\n", encoding="utf-8") + self.sysbin = base / "sysbin" + self.sysbin.mkdir(exist_ok=True) + for name in UTILITIES: + found = shutil.which(name) + if found and not (self.sysbin / name).exists(): + (self.sysbin / name).symlink_to(found) + self.stubs = base / "stubs" + self.pip_log = base / "pip.log" + self.extra_env: dict[str, str] = {} + self.path_dirs: list[Path] = [self.stubs, self.sysbin] + + def stub(self, name: str, text: str, where: Path | None = None) -> Path: + return _executable((where or self.stubs) / name, text) + + def python(self, name: str = "python3", where: Path | None = None) -> Path: + return self.stub(name, FAKE_PYTHON.format(real=sys.executable), where) + + def standard(self, rg: bool = True, pwsh: bool = False) -> "Machine": + self.python() + self.stub("git", "#!/bin/sh\necho 'git version 2.47.1'\n") + if rg: + self.stub("rg", "#!/bin/sh\necho 'ripgrep 14.1.1 (rev 4649aa9700)'\n") + if pwsh: + self.stub("pwsh", "#!/bin/sh\necho 'PowerShell 7.6.6'\n") + return self + + def run(self, *args: str, shell: str) -> subprocess.CompletedProcess: + env = { + "PATH": os.pathsep.join(str(d) for d in self.path_dirs), + "HOME": str(self.base), + "PIP_LOG": str(self.pip_log), + **self.extra_env, + } + return subprocess.run( + [shell, str(self.tools / "preflight.sh"), *args], + capture_output=True, text=True, env=env, timeout=60, + ) + + @property + def tools_file(self) -> Path: + return self.root / toolpaths.FILE_NAME + + def recorded(self) -> dict: + return json.loads(self.tools_file.read_text(encoding="utf-8")) + + +def assert_guidance(output: str, *fragments: str) -> None: + """Every exit-42 stop carries the block: what, why, the fixing command, what next.""" + assert "STOP" in output + for marker in ("Why:", "Fix:", "Next:"): + assert marker in output, f"no {marker!r} in:\n{output}" + for fragment in fragments: + assert fragment in output, f"no {fragment!r} in:\n{output}" + + +@pytest.fixture +def machine(tmp_path: Path) -> Machine: + return Machine(tmp_path.resolve()) + + +pytestmark = pytest.mark.skipif(not SHELLS, reason="no POSIX shell on this machine") + + +# --- the happy path --------------------------------------------------------------- + + +@pytest.mark.parametrize("shell", SHELLS) +def test_complete_path_records_absolute_paths_and_sets_up_the_venv(machine, shell): + machine.standard() + result = machine.run(shell=shell) + assert result.returncode == 0, result.stdout + result.stderr + data = machine.recorded() + assert data["schema"] == 1 and data["complete"] is True + assert set(data["tools"]) == {"python", "git", "rg"} + for path in data["tools"].values(): + assert os.path.isabs(path) and os.path.isfile(path) + assert data["tools"]["rg"] == str(machine.stubs / "rg") + assert (machine.tools / ".venv" / "bin" / "python").is_file() + assert machine.pip_log.read_text(encoding="utf-8").count("install") == 1 + assert "Preflight passed" in result.stdout + + +@pytest.mark.parametrize("shell", SHELLS) +def test_second_run_changes_nothing(machine, shell): + machine.standard() + assert machine.run(shell=shell).returncode == 0 + before = machine.tools_file.read_text(encoding="utf-8") + again = machine.run(shell=shell) + assert again.returncode == 0, again.stdout + assert machine.tools_file.read_text(encoding="utf-8") == before + assert machine.pip_log.read_text(encoding="utf-8").count("install") == 1 + + +# --- stop cases ------------------------------------------------------------------- + + +@pytest.mark.parametrize("shell", SHELLS) +def test_missing_rg_stops_with_42_and_a_missing_line(machine, shell): + machine.standard(rg=False) + result = machine.run(shell=shell) + assert result.returncode == 42 + assert any(line.split()[:2] == ["MISSING", "rg"] for line in result.stdout.splitlines()) + assert_guidance(result.stdout, "ripgrep (rg) was not found", "--set rg=") + # What was found is kept for the next round of the loop, but not as ready. + data = machine.recorded() + assert data["complete"] is False and "rg" not in data["tools"] + assert not (machine.tools / ".venv").exists() + + +@pytest.mark.parametrize("shell", SHELLS) +def test_python_too_old_stops(machine, shell): + machine.standard() + machine.extra_env["FAKE_PY_VERSION"] = "3.9" + result = machine.run(shell=shell) + assert result.returncode == 42 + assert "TOO_OLD" in result.stdout + assert_guidance(result.stdout, "Python 3.9 is too old") + + +@pytest.mark.parametrize("shell", SHELLS) +def test_invalid_set_path_writes_nothing(machine, shell): + machine.standard() + result = machine.run("--set", f"rg={machine.base / 'nowhere' / 'rg'}", shell=shell) + assert result.returncode == 42 + assert_guidance(result.stdout, "The path given for ripgrep (rg) does not work") + assert not machine.tools_file.exists() + + +@pytest.mark.parametrize("shell", SHELLS) +def test_invalid_set_path_leaves_an_existing_file_alone(machine, shell): + machine.standard() + assert machine.run(shell=shell).returncode == 0 + before = machine.tools_file.read_text(encoding="utf-8") + result = machine.run("--set", "git=/nowhere/git", shell=shell) + assert result.returncode == 42 + assert machine.tools_file.read_text(encoding="utf-8") == before + + +@pytest.mark.parametrize("shell", SHELLS) +def test_valid_set_path_is_recorded_and_kept(machine, shell): + machine.standard(rg=False) + elsewhere = machine.stub("rg", "#!/bin/sh\necho 'ripgrep 14.1.1'\n", machine.base / "opt") + result = machine.run("--set", f"rg={elsewhere}", shell=shell) + assert result.returncode == 0, result.stdout + assert machine.recorded()["tools"]["rg"] == str(elsewhere) + # The next run finds it in the file, although it is still not on PATH. + again = machine.run(shell=shell) + assert again.returncode == 0, again.stdout + assert machine.recorded()["tools"]["rg"] == str(elsewhere) + + +@pytest.mark.parametrize("shell", SHELLS) +def test_set_for_a_tool_this_platform_does_not_need_is_a_usage_error(machine, shell): + machine.standard() + result = machine.run("--set", "nonsense=/bin/true", shell=shell) + assert result.returncode == 1 + assert not machine.tools_file.exists() + + +@pytest.mark.parametrize("shell", SHELLS) +def test_venv_that_cannot_be_created_stops(machine, shell): + machine.standard() + machine.extra_env["FAKE_VENV_FAIL"] = "1" + result = machine.run(shell=shell) + assert result.returncode == 42 + assert_guidance(result.stdout, "tools/.venv) could not be created", "ensurepip is not available") + assert machine.recorded()["complete"] is False + + +@pytest.mark.parametrize("shell", SHELLS) +def test_pip_failure_stops_and_is_retried_next_time(machine, shell): + machine.standard() + machine.extra_env["FAKE_PIP_FAIL"] = "1" + result = machine.run(shell=shell) + assert result.returncode == 42 + assert_guidance(result.stdout, "could not be installed into tools/.venv", "network unreachable") + assert machine.recorded()["complete"] is False + del machine.extra_env["FAKE_PIP_FAIL"] + assert machine.run(shell=shell).returncode == 0 + assert machine.pip_log.read_text(encoding="utf-8").count("install") == 2 + + +# --- the store alias ---------------------------------------------------------------- + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("platform, alias", [("linux", "python3"), ("windows", "python")]) +def test_store_alias_is_never_run_and_never_recorded(machine, shell, platform, alias): + marker = machine.base / "alias-was-run" + apps = machine.base / "Users" / "u" / "AppData" / "Local" / "Microsoft" / "WindowsApps" + machine.stub(alias, f"#!/bin/sh\necho ran > '{marker}'\nexit 9009\n", apps) + machine.path_dirs.insert(0, apps) + machine.standard(pwsh=True) + machine.extra_env.update({"CHEMENU_PREFLIGHT_PLATFORM": platform, "CHEMENU_PREFLIGHT_LONGPATHS": "1"}) + if platform == "windows": + machine.python("python") # the real one, behind the alias on PATH + result = machine.run(shell=shell) + assert result.returncode == 0, result.stdout + recorded = machine.recorded()["tools"]["python"] + assert "WindowsApps" not in recorded + assert recorded.startswith(str(machine.stubs)) + assert not marker.exists() + + +@pytest.mark.parametrize("shell", SHELLS) +def test_windows_needs_pwsh_too(machine, shell): + machine.standard(pwsh=False) + machine.extra_env.update({"CHEMENU_PREFLIGHT_PLATFORM": "windows", "CHEMENU_PREFLIGHT_LONGPATHS": "1"}) + result = machine.run(shell=shell) + assert result.returncode == 42 + assert_guidance(result.stdout, "PowerShell 7 (pwsh) was not found") + + +# --- install folder length (D32) -------------------------------------------------- + + +def _machine_with_root_of(tmp_path: Path, length: int) -> Machine: + base = tmp_path.resolve() + name_length = length - len(str(base)) - 1 + assert name_length > 0, "tmp_path is too long for this test" + machine = Machine(base, "r" * name_length) + assert len(str(machine.root)) == length + machine.standard(pwsh=True) + machine.extra_env["CHEMENU_PREFLIGHT_PLATFORM"] = "windows" + return machine + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("length, longpaths, expected", [ + (95, "0", 0), + (96, "0", 42), + (96, "1", 0), +]) +def test_install_folder_limit_at_the_boundary(tmp_path, shell, length, longpaths, expected): + machine = _machine_with_root_of(tmp_path, length) + machine.extra_env["CHEMENU_PREFLIGHT_LONGPATHS"] = longpaths + result = machine.run(shell=shell) + assert result.returncode == expected, result.stdout + if expected == 42: + assert_guidance(result.stdout, f"too long ({length} characters, at most 95)", "C:\\Chemenu") + assert not (machine.tools / ".venv").exists() + + +def test_manifest_limit_and_path_budget_fit_max_path_together(): + """95 + separator + 160 must stay within Windows' 259 - the two numbers are + one sum, kept in two places (tools/prerequisites.txt, titles.PATH_BUDGET).""" + limit = prerequisites.load_manifest().limits["install_dir_max"] + assert limit + 1 + titles.PATH_BUDGET <= 259 + + +def test_manifest_names_the_tools_the_stack_starts(): + manifest = prerequisites.load_manifest() + assert [t.name for t in manifest.tools_for("linux")] == ["python", "git", "rg"] + assert [t.name for t in manifest.tools_for("windows")] == ["python", "git", "rg", "pwsh"] + + +# --- the launcher ----------------------------------------------------------------- + + +def _launch(machine: Machine, *args: str) -> subprocess.CompletedProcess: + env = {"PATH": os.pathsep.join(str(d) for d in machine.path_dirs), "HOME": str(machine.base)} + return subprocess.run([str(machine.tools / "wikitool"), *args], + capture_output=True, text=True, env=env, timeout=30) + + +ECHO_PYTHON = "#!/bin/sh\nprintf '%s\\n' \"$@\"\n" + + +def _complete_file(machine: Machine, complete: bool = True) -> None: + machine.tools_file.write_text(json.dumps( + {"schema": 1, "complete": complete, "tools": {"git": "/usr/bin/git"}}, indent=2), + encoding="utf-8") + + +@pytest.mark.parametrize("state", ["no file", "no venv", "incomplete"]) +def test_launcher_stops_with_42_until_the_preflight_has_passed(machine, state): + if state != "no file": + _complete_file(machine, complete=(state != "incomplete")) + if state != "no venv": + machine.stub("python", ECHO_PYTHON, machine.tools / ".venv" / "bin") + result = _launch(machine, "doctor") + assert result.returncode == 42 + assert "tools/preflight.sh" in result.stderr + assert "Traceback" not in result.stdout + result.stderr + + +@pytest.mark.parametrize("layout", [("bin", "python"), ("Scripts", "python.exe")]) +def test_launcher_runs_the_entry_script_with_either_venv_layout(machine, layout): + _complete_file(machine) + machine.stub(layout[1], ECHO_PYTHON, machine.tools / ".venv" / layout[0]) + result = _launch(machine, "doctor", "--json") + assert result.returncode == 0, result.stderr + assert result.stdout.splitlines() == [str(machine.tools / "run_wikitool.py"), "doctor", "--json"] + + +def test_there_is_a_powershell_launcher_slot_but_no_cmd(): + assert (TOOLS / "wikitool").is_file() + assert not (TOOLS / "wikitool.cmd").exists() + + +# --- trace-hook ------------------------------------------------------------------- + + +def _hook(machine: Machine, *args: str) -> subprocess.CompletedProcess: + env = {"PATH": os.pathsep.join(str(d) for d in machine.path_dirs), "HOME": str(machine.base)} + return subprocess.run([str(machine.tools / "trace-hook"), *args], + capture_output=True, text=True, env=env, timeout=30) + + +@pytest.mark.parametrize("layout", [("bin", "python"), ("Scripts", "python.exe")]) +def test_trace_hook_runs_trace_ingest_with_the_venv_python(machine, layout): + machine.stub(layout[1], ECHO_PYTHON, machine.tools / ".venv" / layout[0]) + result = _hook(machine, "--source", "claude-code", "--event", "prompt.submitted") + assert result.returncode == 0 + assert result.stdout.splitlines() == [ + str(machine.tools / "trace_ingest.py"), "--source", "claude-code", "--event", "prompt.submitted", + ] + + +def test_trace_hook_is_silent_without_a_venv(machine): + result = _hook(machine, "--source", "claude-code") + assert result.returncode == 0 + assert result.stdout == "" and result.stderr == "" + + +def _hook_commands() -> list[str]: + root = config._PACKAGE_ROOT + commands = [] + claude = json.loads((root / ".claude" / "settings.json").read_text(encoding="utf-8")) + for entries in claude["hooks"].values(): + for entry in entries: + commands += [hook["command"] for hook in entry["hooks"]] + copilot = json.loads((root / ".github" / "hooks" / "wiki-trace.json").read_text(encoding="utf-8")) + for entries in copilot["hooks"].values(): + for hook in entries: + commands += [hook["bash"], hook["powershell"]] + import tomllib + + vibe = tomllib.loads((root / ".vibe" / "hooks.toml").read_text(encoding="utf-8")) + commands += [hook["command"] for hook in vibe["hooks"]] + return commands + + +def test_no_hook_relies_on_a_shebang_or_a_bare_python(): + commands = _hook_commands() + assert len(commands) > 10 + for command in commands: + assert command.startswith(("./tools/trace-hook ", ".\\tools\\.venv\\Scripts\\python.exe ")), command + + +# --- toolpaths -------------------------------------------------------------------- + + +def _point_at(monkeypatch, path: Path) -> Path: + monkeypatch.setattr(toolpaths, "tools_file", lambda: path) + return path + + +def test_no_file_means_the_bare_name(tmp_path, monkeypatch): + _point_at(monkeypatch, tmp_path / toolpaths.FILE_NAME) + assert toolpaths.git() == "git" and toolpaths.rg() == "rg" + + +def test_recorded_path_is_used(tmp_path, monkeypatch): + rg = _executable(tmp_path / "bin" / "rg", "#!/bin/sh\n") + _point_at(monkeypatch, tmp_path / toolpaths.FILE_NAME).write_text( + json.dumps({"schema": 1, "complete": True, "tools": {"rg": str(rg)}}), encoding="utf-8") + assert toolpaths.rg() == str(rg) + + +@pytest.mark.parametrize("content, message", [ + ('{"schema": 1, "complete": true, "tools": {}}', "not recorded"), + ('{"schema": 1, "complete": true, "tools": {"git": "/nowhere/git"}}', "no longer exists"), + ("{not json", "not valid JSON"), + ('{"schema": 2, "tools": {}}', "shape this version reads"), +]) +def test_a_present_but_unusable_file_never_falls_back_to_path(tmp_path, monkeypatch, content, message): + _point_at(monkeypatch, tmp_path / toolpaths.FILE_NAME).write_text(content, encoding="utf-8") + with pytest.raises(toolpaths.ToolPathError, match=message) as caught: + toolpaths.git() + assert "tools/preflight.sh" in str(caught.value) + assert not isinstance(caught.value, OSError) + + +def _logging_wrapper(directory: Path, name: str, log: Path) -> Path: + real = shutil.which(name) + assert real, f"{name} is needed for this test" + return _executable(directory / name, f"#!/bin/sh\necho \"$@\" >> '{log}'\nexec '{real}' \"$@\"\n") + + +def test_git_and_rg_start_from_the_recorded_paths_not_from_path(tmp_path, monkeypatch): + log = tmp_path / "calls.log" + hidden = tmp_path / "not-on-path" + git = _logging_wrapper(hidden, "git", log) + rg = _logging_wrapper(hidden, "rg", log) + _point_at(monkeypatch, tmp_path / toolpaths.FILE_NAME).write_text(json.dumps( + {"schema": 1, "complete": True, "tools": {"git": str(git), "rg": str(rg)}}), encoding="utf-8") + + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Recorded Git"], cwd=repo, check=True) + (repo / "kb").mkdir() + (repo / "kb" / "Page.md").write_text("---\ntitle: Page\n---\nkingfisher\n", encoding="utf-8") + monkeypatch.setattr(config, "ROOT", repo) + monkeypatch.setenv("PATH", str(tmp_path / "empty")) + + assert config.default_author() == "Recorded Git" + + from chemenu.search.base import SearchQuery + from chemenu.search.ripgrep import RipgrepBackend + backend = RipgrepBackend(search_root=repo / "kb", repo_root=repo) + backend.search(SearchQuery(text="kingfisher"), {}) + + calls = log.read_text(encoding="utf-8") + assert "config user.name" in calls + assert "kingfisher" in calls + + +def test_cli_turns_a_tool_path_error_into_an_error_line(tmp_path, monkeypatch, capsys): + from chemenu import cli + + _point_at(monkeypatch, tmp_path / toolpaths.FILE_NAME).write_text( + '{"schema": 1, "complete": true, "tools": {}}', encoding="utf-8") + monkeypatch.setattr(sys, "argv", ["wikitool", "search", "kingfisher"]) + with pytest.raises(SystemExit) as exited: + cli._run_traced("search", ["kingfisher"]) + assert exited.value.code == 1 + out = capsys.readouterr().out + assert "ERROR" in out and "tools/preflight.sh" in out diff --git a/tools/chemenu/toolpaths.py b/tools/chemenu/toolpaths.py new file mode 100644 index 0000000..cf513e0 --- /dev/null +++ b/tools/chemenu/toolpaths.py @@ -0,0 +1,88 @@ +"""Where the programs this package starts live: `.wikitool-tools.json`. + +The preflight (`tools/preflight.sh`, `tools/preflight.ps1`) finds `git`, `rg` and +the rest once, checks they run, and records their absolute native paths in this +file beside `tools/`. Everything here that starts one of them asks `resolve()` +for the path instead of trusting whatever `PATH` the calling process inherited - +a harness session on Windows can hold a `PATH` from before the tool was +installed, and then the same machine answers "missing" in one terminal and +"present" in the next. + +The file belongs to the checkout the code runs from, not to the corpus it is +pointed at: `CHEMENU_ROOT` may name another tree, and that tree's tools are this +installation's tools. So it is read from `config._PACKAGE_ROOT`, never `ROOT`. + +Enforcement lives in the launcher, which refuses to start (exit 42) until the +preflight has written a complete file. Here, an absent file falls back to the +bare program name - the case of the test suite and of `python -m chemenu.cli` +run by hand. A file that is present but wrong never falls back: a tool it does +not name, or names at a path that has gone, is a `ToolPathError` that tells the +user to run the preflight again. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Optional + +from chemenu import config +from chemenu.errors import ChemenuError + +FILE_NAME = ".wikitool-tools.json" +SCHEMA = 1 +PREFLIGHT = "run the preflight again: tools/preflight.sh" + + +class ToolPathError(ChemenuError): + """The recorded tool paths cannot be used. Deliberately not an `OSError`: + call sites that treat a failed start as "tool absent" must not swallow + this - the fix is a command for the user, not a degraded answer.""" + + +def tools_file() -> Path: + return config._PACKAGE_ROOT / FILE_NAME + + +def load(path: Optional[Path] = None) -> Optional[dict]: + """The parsed file, or None when it does not exist. Raises `ToolPathError` + for a file that exists but is not one this version writes.""" + path = tools_file() if path is None else path + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError: + return None + except OSError as exc: + raise ToolPathError(f"{FILE_NAME} cannot be read ({exc}) - {PREFLIGHT}") from exc + try: + data = json.loads(text) + except ValueError as exc: + raise ToolPathError(f"{FILE_NAME} is not valid JSON ({exc}) - {PREFLIGHT}") from exc + if not isinstance(data, dict) or data.get("schema") != SCHEMA or not isinstance(data.get("tools"), dict): + raise ToolPathError( + f"{FILE_NAME} is not in the shape this version reads (schema {SCHEMA} with a " + f"`tools` object) - {PREFLIGHT}" + ) + return data + + +def resolve(name: str) -> str: + """The program to start for `name`: its recorded path, or `name` itself + when no file has been written.""" + data = load() + if data is None: + return name + path = data["tools"].get(name) + if not isinstance(path, str) or not path: + raise ToolPathError(f"`{name}` is not recorded in {FILE_NAME} - {PREFLIGHT}") + if not os.path.isfile(path): + raise ToolPathError(f"`{name}` is recorded at {path}, which no longer exists - {PREFLIGHT}") + return path + + +def git() -> str: + return resolve("git") + + +def rg() -> str: + return resolve("rg") diff --git a/tools/preflight.sh b/tools/preflight.sh new file mode 100755 index 0000000..0fd1162 --- /dev/null +++ b/tools/preflight.sh @@ -0,0 +1,520 @@ +#!/bin/sh +# Preflight: check that this machine has what the stack needs, record where each +# tool lives, and set up tools/.venv - before any `wikitool` command can run. +# +# tools/preflight.sh check, record, set up +# tools/preflight.sh --set rg=/opt/rg/rg use a path the user named +# +# Exit 0: everything is in place and .wikitool-tools.json is complete. +# Exit 42: the user has to act. The output says what, why, the command that fixes +# it and what happens next; show it verbatim and wait (AGENTS.md § Tool +# error contract). Never install anything on the user's behalf. +# Exit 1: this script was called wrongly, or the tree next to it is incomplete. +# +# POSIX sh on purpose: it has to run before Python is known to exist, under dash, +# bash and the Git Bash that Claude Code uses on Windows. The PowerShell +# counterpart is tools/preflight.ps1; what both check is tools/prerequisites.txt. +# The procedure around this script: instructions/preflight.md. +# +# Two variables exist for the test suite and are read nowhere else: +# CHEMENU_PREFLIGHT_PLATFORM (windows|macos|linux) and +# CHEMENU_PREFLIGHT_LONGPATHS (0|1) stand in for `uname` and the registry. + +set -u + +DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P) +ROOT=$(dirname -- "$DIR") +MANIFEST="$DIR/prerequisites.txt" +TOOLS_FILE="$ROOT/.wikitool-tools.json" +VENV="$DIR/.venv" +REQUIREMENTS="$DIR/requirements.txt" +STAMP="$VENV/.chemenu-requirements.sha256" +SELF="tools/preflight.sh" + +PYPROBE='import sys; print("%d.%d" % sys.version_info[:2]); print(sys.executable)' + +usage() { + cat <<'EOF' +usage: tools/preflight.sh [--set =]... + +Checks the tools this stack needs (tools/prerequisites.txt), records their paths +in .wikitool-tools.json and sets up tools/.venv. Exit 0 means ready; exit 42 +means the user has to act - the output says how. +EOF +} + +# --- arguments ---------------------------------------------------------------- + +SETS="" +while [ $# -gt 0 ]; do + case "$1" in + --set) + if [ $# -lt 2 ]; then + echo "preflight: --set needs =" >&2 + exit 1 + fi + SETS="$SETS +$2" + shift + ;; + --set=*) SETS="$SETS +${1#--set=}" ;; + -h|--help) usage; exit 0 ;; + *) echo "preflight: unknown argument: $1" >&2; usage >&2; exit 1 ;; + esac + shift +done + +if [ ! -f "$MANIFEST" ]; then + echo "preflight: $MANIFEST is missing - this script has to run from inside an unpacked stack tree." >&2 + exit 1 +fi + +# --- platform ----------------------------------------------------------------- + +if [ -n "${CHEMENU_PREFLIGHT_PLATFORM:-}" ]; then + PLATFORM=$CHEMENU_PREFLIGHT_PLATFORM +else + case "$(uname -s 2>/dev/null)" in + MINGW*|MSYS*|CYGWIN*) PLATFORM=windows ;; + Darwin) PLATFORM=macos ;; + *) PLATFORM=linux ;; + esac +fi + +# A path as the rest of the machine spells it: under Git Bash `/c/Tools/rg.exe` +# becomes `C:\Tools\rg.exe`, which is what Python and PowerShell read back. +native_path() { + if [ "$PLATFORM" = windows ] && command -v cygpath >/dev/null 2>&1; then + cygpath -w "$1" + else + printf '%s\n' "$1" + fi +} + +# --- problems and the guidance block -------------------------------------------- + +PROBLEMS=0 +GUIDE="" +BAD_SET=0 + +# problem +problem() { + PROBLEMS=$((PROBLEMS + 1)) + fix=$(printf '%s\n' "$3" | sed '2,$s/^/ /') + GUIDE="$GUIDE +$PROBLEMS) $1 + Why: $2 + Fix: $fix + Next: Tell the agent once this is done - it runs this check again. +" +} + +stop() { + if [ "$PROBLEMS" -eq 1 ]; then + noun="1 thing needs" + else + noun="$PROBLEMS things need" + fi + echo "" + echo "STOP - $noun your attention before this wiki can run." + echo "(Agent: show this output to the user exactly as it is, then wait. Do not install" + echo "anything yourself and do not work around it.)" + printf '%s' "$GUIDE" + exit 42 +} + +# --- the manifest --------------------------------------------------------------- + +manifest_field() { # + while IFS='|' read -r kind name rest; do + case "$kind" in ''|'#'*) continue ;; esac + if [ "$kind" = "$1" ] && [ "$name" = "$2" ]; then + printf '%s|%s|%s\n' "$kind" "$name" "$rest" | cut -d'|' -f"$3" + return 0 + fi + done < "$MANIFEST" + return 1 +} + +TOOLS="" +while IFS='|' read -r kind name minimum platforms rest; do + case "$kind" in ''|'#'*) continue ;; esac + [ "$kind" = tool ] || continue + case "$name" in *[!a-z0-9]*|'') echo "preflight: bad tool name '$name' in $MANIFEST" >&2; exit 1 ;; esac + if [ "$platforms" = all ] || [ "$platforms" = "$PLATFORM" ]; then + TOOLS="$TOOLS $name" + fi +done < "$MANIFEST" + +# The install command for this machine's package manager, or every known one. +install_hint() { # + choco=$(manifest_field tool "$1" 7) + winget=$(manifest_field tool "$1" 8) + brew=$(manifest_field tool "$1" 9) + apt=$(manifest_field tool "$1" 10) + pacman=$(manifest_field tool "$1" 11) + hint="" + case "$PLATFORM" in + windows) + if command -v choco >/dev/null 2>&1 && [ "$choco" != - ]; then + hint="$choco (in a terminal opened as administrator)" + elif command -v winget >/dev/null 2>&1 && [ "$winget" != - ]; then + hint=$winget + fi ;; + macos) + if command -v brew >/dev/null 2>&1 && [ "$brew" != - ]; then hint=$brew; fi ;; + *) + if command -v apt-get >/dev/null 2>&1 && [ "$apt" != - ]; then + hint=$apt + elif command -v pacman >/dev/null 2>&1 && [ "$pacman" != - ]; then + hint=$pacman + fi ;; + esac + if [ -n "$hint" ]; then + printf '%s\n' "$hint" + return + fi + printf 'Install it with the package manager this computer uses, for example:\n' + for entry in "choco:$choco" "winget:$winget" "brew:$brew" "apt:$apt" "pacman:$pacman"; do + case "$entry" in *:-) continue ;; esac + printf ' %s\n' "${entry#*:}" + done +} + +# --- version helpers ------------------------------------------------------------ + +# major.minor of the first version-looking token in $1 ("git version 2.47.1" -> 2.47) +version_of() { + printf '%s\n' "$1" | sed -n '1{s/^[^0-9]*\([0-9][0-9]*\)\(\.[0-9][0-9]*\)\{0,1\}.*/\1\2/p;}' +} + +version_ge() { # , both major[.minor] + have=$1 want=$2 + case "$have" in *.*) ;; *) have="$have.0" ;; esac + case "$want" in *.*) ;; *) want="$want.0" ;; esac + hm=${have%%.*} hn=${have#*.} + wm=${want%%.*} wn=${want#*.} + [ "$hm" -gt "$wm" ] || { [ "$hm" -eq "$wm" ] && [ "$hn" -ge "$wn" ]; } +} + +# --- finding tools -------------------------------------------------------------- + +# The Microsoft Store's app-execution aliases: a python.exe that opens the Store +# instead of running anything. Never executed, never recorded. +is_store_alias() { + case "$1" in *[Ww]indows[Aa]pps/*|*[Ww]indows[Aa]pps\\*) return 0 ;; esac + return 1 +} + +# Every executable called on PATH, in PATH order, store aliases dropped. +on_path() { + set -f + old_ifs=$IFS + IFS=: + for d in $PATH; do + [ -n "$d" ] || d=. + for f in "$d/$1" "$d/$1.exe"; do + if [ -f "$f" ] && [ -x "$f" ] && ! is_store_alias "$f"; then + printf '%s\n' "$f" + fi + done + done + IFS=$old_ifs + set +f +} + +usable() { # + [ -n "$1" ] && [ -f "$1" ] && [ -x "$1" ] && ! is_store_alias "$1" +} + +# Runs a Python candidate; prints "\n". +probe_python() { # [extra argument, e.g. -3 for py] + if [ $# -gt 1 ]; then + "$1" "$2" -c "$PYPROBE" 2>/dev/null | tr -d '\r' + else + "$1" -c "$PYPROBE" 2>/dev/null | tr -d '\r' + fi +} + +# The value recorded for in .wikitool-tools.json, unescaped. +recorded() { + [ -f "$TOOLS_FILE" ] || return 1 + value=$(sed -n "s/^ *\"$1\": *\"\(.*\)\",\{0,1\} *\$/\1/p" "$TOOLS_FILE" | head -n 1) + [ -n "$value" ] || return 1 + printf '%s\n' "$value" | sed 's/\\"/"/g; s/\\\\/\\/g' +} + +set_value() { # -> the path given with --set, if any + printf '%s\n' "$SETS" | sed -n "s/^$1=//p" | head -n 1 +} + +# --- --set: every named path has to work, or nothing is written ------------------- + +for entry in $(printf '%s\n' "$SETS" | sed -n 's/^\([^=]*\)=.*/\1/p'); do + case " $TOOLS " in + *" $entry "*) ;; + *) echo "preflight: --set names '$entry', which is not a tool this platform needs:$TOOLS" >&2 + exit 1 ;; + esac +done + +# --- python ------------------------------------------------------------------------- + +PY="" PY_VERSION="" PY_OLD="" +PY_MIN=$(manifest_field tool python 3) + +try_python() { # [extra] -> sets PY/PY_VERSION on success, PY_OLD when too old + out=$(probe_python "$@") || return 1 + version=$(printf '%s\n' "$out" | sed -n 1p) + executable=$(printf '%s\n' "$out" | sed -n 2p) + [ -n "$version" ] && [ -n "$executable" ] || return 1 + if version_ge "$version" "$PY_MIN"; then + PY=$executable PY_VERSION=$version + return 0 + fi + PY_OLD="$version $1" + return 1 +} + +python_extra() { # py and py.exe are the launcher: they need -3 + case "${1##*/}" in py|py.exe) printf '%s\n' -3 ;; esac +} + +given=$(set_value python) +if [ -n "$given" ]; then + extra=$(python_extra "$given") + if ! usable "$given" || ! try_python "$given" $extra; then + problem "The path given for Python does not work: $given" \ + "every wikitool command runs on Python $PY_MIN or newer, and this path did not start one" \ + "Check the path - it has to be the python executable itself, version $PY_MIN or newer. +Then run: $SELF --set python=" + BAD_SET=1 + fi +else + previous=$(recorded python) || previous="" + if [ -n "$previous" ] && usable "$previous"; then + try_python "$previous" || true + fi + if [ -z "$PY" ]; then + if [ "$PLATFORM" = windows ]; then + order="python: py:-3 python3:" + else + order="python3: python:" + fi + for candidate in $order; do + name=${candidate%%:*} + extra=${candidate#*:} + found=$(on_path "$name") + [ -n "$found" ] || continue + # A here-document, not a pipe: `break 2` has to leave the outer loop + # of this shell, and a path may contain spaces ("Program Files"). + while IFS= read -r path; do + if [ -n "$extra" ]; then + try_python "$path" "$extra" && break 2 + else + try_python "$path" && break 2 + fi + done < + REPORT="$REPORT$(printf '%-8s %-7s %-8s %s' "$1" "$2" "$3" "$4") +" +} + +if [ -n "$PY" ]; then + report OK python "$PY_VERSION" "$PY" + eval "FOUND_python=\$PY" +elif [ "$BAD_SET" -eq 0 ]; then + label=$(manifest_field tool python 5) + why=$(manifest_field tool python 6) + if [ -n "$PY_OLD" ]; then + report TOO_OLD python "${PY_OLD%% *}" "${PY_OLD#* }" + problem "$label ${PY_OLD%% *} is too old - this stack needs $PY_MIN or newer." "$why" \ + "$(install_hint python) +Or, if a newer one is already installed, give its full path: +$SELF --set python=" + else + report MISSING python - - + problem "$label $PY_MIN or newer was not found." "$why" \ + "$(install_hint python) +Or, if it is installed somewhere this check did not look, give its full path: +$SELF --set python=" + fi +fi + +for tool in $TOOLS; do + [ "$tool" = python ] && continue + label=$(manifest_field tool "$tool" 5) + why=$(manifest_field tool "$tool" 6) + minimum=$(manifest_field tool "$tool" 3) + given=$(set_value "$tool") + path="" + if [ -n "$given" ]; then + if usable "$given"; then + path=$given + else + problem "The path given for $label does not work: $given" "$why" \ + "Check the path - it has to be the $tool executable itself. +Then run: $SELF --set $tool=" + BAD_SET=1 + continue + fi + else + previous=$(recorded "$tool") || previous="" + if [ -n "$previous" ] && usable "$previous"; then + path=$previous + else + path=$(on_path "$tool" | head -n 1) + fi + fi + if [ -z "$path" ]; then + report MISSING "$tool" - - + problem "$label was not found." "$why" \ + "$(install_hint "$tool") +Or, if it is installed somewhere this check did not look, give its full path: +$SELF --set $tool=" + continue + fi + version=$(version_of "$("$path" --version 2>/dev/null | tr -d '\r')") + if [ "$minimum" != - ] && { [ -z "$version" ] || ! version_ge "$version" "$minimum"; }; then + report TOO_OLD "$tool" "${version:-unknown}" "$path" + problem "$label ${version:-of unknown version} is too old - this stack needs $minimum or newer." "$why" \ + "$(install_hint "$tool")" + continue + fi + native=$(native_path "$path") + report OK "$tool" "${version:--}" "$native" + eval "FOUND_$tool=\$native" +done + +# --- install folder length (Windows, long paths off) ------------------------------ + +longpaths_enabled() { + if [ -n "${CHEMENU_PREFLIGHT_LONGPATHS:-}" ]; then + [ "$CHEMENU_PREFLIGHT_LONGPATHS" = 1 ] + return + fi + # MSYS would rewrite the `/v` into a path without the exclusion. An unreadable + # key counts as "off": the limit then protects a machine it did not have to. + answer=$(MSYS2_ARG_CONV_EXCL='*' reg query 'HKLM\SYSTEM\CurrentControlSet\Control\FileSystem' \ + /v LongPathsEnabled 2>/dev/null) || return 1 + case "$answer" in *0x1*) return 0 ;; esac + return 1 +} + +# Length in UTF-16 code units, which is what MAX_PATH counts. +utf16_length() { + if command -v iconv >/dev/null 2>&1; then + bytes=$(printf '%s' "$1" | iconv -f UTF-8 -t UTF-16LE 2>/dev/null | wc -c | tr -d ' ') + if [ "${bytes:-0}" -gt 0 ]; then + echo $((bytes / 2)) + return + fi + fi + echo ${#1} +} + +if [ "$PLATFORM" = windows ] && ! longpaths_enabled; then + limit=$(manifest_field limit install_dir_max 3) + folder=$(native_path "$ROOT") + length=$(utf16_length "$folder") + if [ "$length" -gt "$limit" ]; then + problem "The folder this wiki is installed in is too long ($length characters, at most $limit): $folder" \ + "Windows on this computer only allows paths of up to 259 characters, and the wiki's own files need the rest" \ + "Move the wiki to a shorter folder, for example C:\\Chemenu, and run this check there. +Alternatively, someone with administrator rights can turn on long paths in Windows." + fi +fi + +# --- record what was found ---------------------------------------------------------- + +json_string() { + printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' +} + +write_tools_file() { # + tmp="$TOOLS_FILE.tmp.$$" + { + printf '{\n "schema": 1,\n "complete": %s,\n "tools": {' "$1" + separator="" + for tool in $TOOLS; do + eval "value=\${FOUND_$tool:-}" + [ -n "$value" ] || continue + printf '%s\n "%s": "%s"' "$separator" "$tool" "$(json_string "$value")" + separator="," + done + printf '\n }\n}\n' + } > "$tmp" && mv -f "$tmp" "$TOOLS_FILE" +} + +printf '%s' "$REPORT" + +if [ "$BAD_SET" -eq 1 ]; then + stop # nothing is written for a path that does not work +fi +write_tools_file false +if [ "$PROBLEMS" -gt 0 ]; then + stop +fi + +# --- tools/.venv ---------------------------------------------------------------------- + +venv_python() { + if [ -x "$VENV/bin/python" ]; then + printf '%s\n' "$VENV/bin/python" + elif [ -f "$VENV/Scripts/python.exe" ]; then + printf '%s\n' "$VENV/Scripts/python.exe" + else + return 1 + fi +} + +last_lines() { + printf '%s\n' "$1" | tail -n 5 | sed 's/^/ /' +} + +VPY=$(venv_python) || VPY="" +if [ -z "$VPY" ] || ! "$VPY" -m pip --version >/dev/null 2>&1; then + if ! output=$("$PY" -m venv --clear "$VENV" 2>&1) || ! VPY=$(venv_python); then + fix="Python said: +$(last_lines "$output")" + if [ "$PLATFORM" = linux ] && command -v apt-get >/dev/null 2>&1; then + fix="sudo apt install python3-venv +$fix" + fi + problem "The wiki's own Python environment (tools/.venv) could not be created." \ + "wikitool runs in that environment, so that nothing it installs touches the rest of the computer" \ + "$fix" + stop + fi +fi + +want=$("$VPY" -c 'import hashlib, sys; print(hashlib.sha256(open(sys.argv[1], "rb").read()).hexdigest())' \ + "$REQUIREMENTS" 2>/dev/null | tr -d '\r') +have=$(cat "$STAMP" 2>/dev/null) || have="" +if [ -z "$want" ] || [ "$want" != "$have" ]; then + if ! output=$("$VPY" -m pip install --disable-pip-version-check --quiet -r "$REQUIREMENTS" 2>&1); then + problem "The libraries wikitool needs could not be installed into tools/.venv." \ + "they are downloaded once from the internet; without them no wikitool command starts" \ + "Check that this computer can reach the internet (a proxy or a security program can block it; if so, ask whoever looks after this computer). +pip said: +$(last_lines "$output")" + stop + fi + printf '%s\n' "$want" > "$STAMP" +fi +echo "OK venv - $(native_path "$VENV")" + +write_tools_file true +echo "" +echo "Preflight passed. Tool paths are recorded in .wikitool-tools.json; tools/wikitool is ready." +exit 0 diff --git a/tools/prerequisites.txt b/tools/prerequisites.txt new file mode 100644 index 0000000..454a2c6 --- /dev/null +++ b/tools/prerequisites.txt @@ -0,0 +1,22 @@ +# What this stack needs on the machine, read by tools/preflight.sh, tools/preflight.ps1 +# and `wikitool doctor` - one list, so the three cannot disagree. The procedure around it +# is instructions/preflight.md. +# +# Line format, fields separated by `|` so a POSIX shell reads it with `IFS='|' read`: +# +# limit|| +# tool||||