raw: fetch behind a login - raw login stores the user's session as a secret outside the tree, fetch.routes sends a host through it or to an external fetcher, a handler's render goes through the optional browser component (preflight --with browser) (#169); 10.0.0-beta.8

Files changed:
- CHANGES.md
- INSTALL.md
- VERSION
- docs/how-to/README.md
- docs/how-to/fetch-pages-behind-a-login.md
- docs/how-to/ingest-a-source.md
- docs/reference/commands/README.md
- docs/reference/commands/instance-health.md
- docs/reference/commands/raw-material-and-uploads.md
- docs/reference/configuration.md
- docs/reference/exit-codes.md
- instructions/preflight.md
- instructions/wiki-ingest/SKILL.md
- raw/CONTRACT.md
- sites/CONTRACT.md
- tools/CONTRACT.md
- tools/README.md
- tools/chemenu/browser.py
- tools/chemenu/cli_contract.py
- tools/chemenu/commands/doctor.py
- tools/chemenu/commands/raw/__init__.py
- tools/chemenu/commands/raw/fetch.py
- tools/chemenu/commands/raw/login.py
- tools/chemenu/config_shapes.py
- tools/chemenu/fetch_routes.py
- tools/chemenu/reference_doc.py
- tools/chemenu/settings.py
- tools/chemenu/sites/__init__.py
- tools/chemenu/sites/data.py
- tools/chemenu/sites/site.schema.json
- tools/chemenu/tests/test_cli.py
- tools/chemenu/tests/test_config_shapes.py
- tools/chemenu/tests/test_doctor.py
- tools/chemenu/tests/test_preflight.py
- tools/chemenu/tests/test_preflight_pwsh.py
- tools/chemenu/tests/test_raw_fetch_routes.py
- tools/chemenu/tests/test_raw_login.py
- tools/chemenu/tests/test_site_files.py
- tools/chemenu/web_capture.py
- tools/chemenu/web_session.py
- tools/preflight.ps1
- tools/preflight.sh
- tools/requirements-browser.txt

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2

tools/preflight.sh

@@ -4,6 +4,12 @@
 #
 #   tools/preflight.sh                      check, record, set up
 #   tools/preflight.sh --set rg=/opt/rg/rg  use a path the user named
+#   tools/preflight.sh --with browser       also install the optional browser component
+#
+# --with browser installs tools/requirements-browser.txt into tools/.venv and the
+# Chromium that Playwright drives into tools/.venv/ms-playwright - only on request,
+# because it is a large download that only `raw login` and pages a site handler
+# declares `render` need. Once installed, every later run keeps it up to date.
 #
 # As the release asset `preflight.sh` - a copy with no tools/prerequisites.txt next
 # to it - the script is the first install step instead: it downloads the stack
@@ -48,6 +54,9 @@ TOOLS_FILE="$ROOT/.wikitool-tools.json"
 VENV="$DIR/.venv"
 REQUIREMENTS="$DIR/requirements.txt"
 STAMP="$VENV/.chemenu-requirements.sha256"
+BROWSER_REQUIREMENTS="$DIR/requirements-browser.txt"
+BROWSER_STAMP="$VENV/.chemenu-browser.sha256"
+BROWSERS="$VENV/ms-playwright"
 SELF="tools/preflight.sh"
 
 # Filled in by the release build, in the copy attached to the release; empty in a tree.
@@ -58,12 +67,13 @@ PYPROBE='import sys; print("%d.%d" % sys.version_info[:2]); print(sys.executable
 
 usage() {
     cat <<'EOF'
-usage: tools/preflight.sh [--set <tool>=<path>]...
-       preflight.sh [--into <path>] [--archive <tarball>] [--set <tool>=<path>]...
+usage: tools/preflight.sh [--set <tool>=<path>]... [--with browser]
+       preflight.sh [--into <path>] [--archive <tarball>] [--set <tool>=<path>]... [--with browser]
 
 Checks the tools this stack needs (tools/prerequisites.txt), records their paths
 in .wikitool-tools.json and sets up tools/.venv. Exit 0 means ready; exit 42
-means the user has to act - the output says how.
+means the user has to act - the output says how. --with browser also installs
+the optional browser component (Playwright and its Chromium) into tools/.venv.
 
 The second form is the release asset: it downloads the release (or takes
 --archive), checks its sha256, unpacks it into its own folder or --into (empty,
@@ -75,8 +85,25 @@ EOF
 
 SETS=""
 INTO="" ARCHIVE=""
+WITH=""
+with_component() { # <name>
+    case "$1" in
+        browser) WITH=$1 ;;
+        *) echo "preflight: --with names '$1', which is not an optional component (known: browser)" >&2
+           exit 1 ;;
+    esac
+}
 while [ $# -gt 0 ]; do
     case "$1" in
+        --with)
+            if [ $# -lt 2 ] || [ -z "$2" ]; then
+                echo "preflight: --with needs a component name (known: browser)" >&2
+                exit 1
+            fi
+            with_component "$2"
+            shift
+            ;;
+        --with=*) with_component "${1#--with=}" ;;
         --into|--archive)
             if [ $# -lt 2 ] || [ -z "$2" ]; then
                 echo "preflight: $1 needs a path" >&2
@@ -426,6 +453,7 @@ ${entry##*/}"
     done <<EOT
 $SETS
 EOT
+    [ -z "$WITH" ] || set -- "$@" --with "$WITH"
     trap - EXIT
     cleanup
     exec /bin/sh "$TARGET/tools/preflight.sh" "$@"
@@ -842,6 +870,48 @@ fi
 echo "OK       venv     -        $(native_path "$VENV")"
 
 write_tools_file true
+
+# --- optional components: installed on request, then kept up to date ---------------
+# After .wikitool-tools.json is complete on purpose: a component that fails to install
+# stops this run with exit 42, but leaves every wikitool command working.
+
+has_browser() {
+    for entry in "$BROWSERS"/chromium*; do
+        [ -e "$entry" ] && return 0
+    done
+    return 1
+}
+
+if [ "$WITH" = browser ] || [ -f "$BROWSER_STAMP" ]; then
+    if [ ! -f "$BROWSER_REQUIREMENTS" ]; then
+        echo "preflight: $BROWSER_REQUIREMENTS is missing - the stack tree next to this script is incomplete." >&2
+        exit 1
+    fi
+    want=$("$VPY" -c 'import hashlib, sys; print(hashlib.sha256(open(sys.argv[1], "rb").read()).hexdigest())' \
+        "$BROWSER_REQUIREMENTS" 2>/dev/null | tr -d '\r')
+    have=$(cat "$BROWSER_STAMP" 2>/dev/null) || have=""
+    if [ -z "$want" ] || [ "$want" != "$have" ] || ! has_browser; then
+        if ! output=$("$VPY" -m pip install --disable-pip-version-check --quiet -r "$BROWSER_REQUIREMENTS" 2>&1); then
+            problem "The optional browser component could not be installed into tools/.venv." \
+                "only raw login and pages a site handler fetches through a browser need it; everything else works without it" \
+                "Check that this computer can reach the internet (a proxy or a security program can block it; if so, ask whoever looks after this computer), then run: $SELF --with browser
+pip said:
+$(last_lines "$output")"
+            stop
+        fi
+        if ! output=$(PLAYWRIGHT_BROWSERS_PATH="$BROWSERS" "$VPY" -m playwright install chromium 2>&1); then
+            problem "The browser of the optional browser component could not be downloaded." \
+                "only raw login and pages a site handler fetches through a browser need it; everything else works without it" \
+                "Check that this computer can reach the internet, then run: $SELF --with browser
+Playwright said:
+$(last_lines "$output")"
+            stop
+        fi
+        printf '%s\n' "$want" > "$BROWSER_STAMP"
+    fi
+    echo "OK       browser  -        $(native_path "$BROWSERS")"
+fi
+
 echo ""
 echo "Preflight passed. Tool paths are recorded in .wikitool-tools.json; tools/wikitool is ready."
 exit 0