raw: fetch behind a login - raw login stores the user's session as a secret outside the tree, fetch.routes sends a host through it or to an external fetcher, a handler's render goes through the optional browser component (preflight --with browser) (#169); 10.0.0-beta.8

Files changed:
- CHANGES.md
- INSTALL.md
- VERSION
- docs/how-to/README.md
- docs/how-to/fetch-pages-behind-a-login.md
- docs/how-to/ingest-a-source.md
- docs/reference/commands/README.md
- docs/reference/commands/instance-health.md
- docs/reference/commands/raw-material-and-uploads.md
- docs/reference/configuration.md
- docs/reference/exit-codes.md
- instructions/preflight.md
- instructions/wiki-ingest/SKILL.md
- raw/CONTRACT.md
- sites/CONTRACT.md
- tools/CONTRACT.md
- tools/README.md
- tools/chemenu/browser.py
- tools/chemenu/cli_contract.py
- tools/chemenu/commands/doctor.py
- tools/chemenu/commands/raw/__init__.py
- tools/chemenu/commands/raw/fetch.py
- tools/chemenu/commands/raw/login.py
- tools/chemenu/config_shapes.py
- tools/chemenu/fetch_routes.py
- tools/chemenu/reference_doc.py
- tools/chemenu/settings.py
- tools/chemenu/sites/__init__.py
- tools/chemenu/sites/data.py
- tools/chemenu/sites/site.schema.json
- tools/chemenu/tests/test_cli.py
- tools/chemenu/tests/test_config_shapes.py
- tools/chemenu/tests/test_doctor.py
- tools/chemenu/tests/test_preflight.py
- tools/chemenu/tests/test_preflight_pwsh.py
- tools/chemenu/tests/test_raw_fetch_routes.py
- tools/chemenu/tests/test_raw_login.py
- tools/chemenu/tests/test_site_files.py
- tools/chemenu/web_capture.py
- tools/chemenu/web_session.py
- tools/preflight.ps1
- tools/preflight.sh
- tools/requirements-browser.txt

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2

tools/chemenu/browser.py

@@ -0,0 +1,141 @@
+"""The optional browser component: Playwright and the Chromium it drives.
+
+Two things in the stack need a real browser, and nothing else does: `raw login`,
+where the user signs in in a visible window - captcha and second factor
+included - and a fetch of a page whose site handler declares `render`, because
+the page draws its content with JavaScript. Everything else, a fetch with the
+user's session included, goes through the built-in fetch.
+
+So the component is optional. `tools/requirements-browser.txt` holds the
+package, and the preflight installs it, together with the browser, only when the
+user asks for it (`--with browser`); the agent installs nothing. Playwright is
+imported inside the functions that use it, never at module level, so a venv
+without it runs every other command unchanged. Missing, it is `BrowserMissing`
+- exit 42, since only the user can install it.
+
+The browser lives inside the venv (`browsers_path`), not in Playwright's default
+cache in the home directory: the preflight sets everything it installs up inside
+the install folder, and a second instance on the machine must not share or
+update another's browser.
+"""
+from __future__ import annotations
+
+import importlib.util
+import os
+import sys
+import time
+from pathlib import Path
+from typing import Optional
+
+from chemenu import toolpaths, web_capture
+from chemenu.errors import BackendError, ChemenuError
+
+PACKAGE = "playwright"
+BROWSERS_ENV = "PLAYWRIGHT_BROWSERS_PATH"
+LOGIN_WAIT_SECONDS = 15 * 60
+PREFLIGHT_WITH = f"tools/preflight.sh --with browser (PowerShell 7: {toolpaths.PREFLIGHT_PWSH} --with browser)"
+
+
+class BrowserMissing(ChemenuError):
+    """The browser component is not installed. A human installs it - the
+    agent never does - so the CLI renders this as exit 42."""
+
+
+def browsers_path() -> Path:
+    """Where the preflight puts the browser: inside the venv the tool runs in."""
+    return Path(sys.prefix) / "ms-playwright"
+
+
+def missing() -> Optional[str]:
+    """Why the component cannot be used, or None when it can."""
+    if importlib.util.find_spec(PACKAGE) is None:
+        return f"the Python package `{PACKAGE}` is not installed in the venv"
+    directory = browsers_path()
+    if not directory.is_dir() or not any(p.name.startswith("chromium") for p in directory.iterdir()):
+        return f"its browser is not installed (looked in {directory})"
+    return None
+
+
+def require(purpose: str) -> None:
+    """Raise `BrowserMissing` naming `purpose` and the preflight option."""
+    why = missing()
+    if why is not None:
+        raise BrowserMissing(
+            f"{purpose} needs the optional browser component, and {why}. Ask the user to install it: "
+            f"{PREFLIGHT_WITH}. Only the user installs it; nothing was written."
+        )
+
+
+def _environment() -> None:
+    os.environ.setdefault(BROWSERS_ENV, str(browsers_path()))
+
+
+def login(url: str, wait_seconds: float = LOGIN_WAIT_SECONDS) -> dict:
+    """Open `url` in a visible browser window and wait until the user closes
+    it, then return the browser state (cookies and storage). The state is read
+    once a second while the window is open, so closing the window - or the
+    whole browser - keeps what the last reading saw."""
+    require("`raw login`")
+    _environment()
+    from playwright.sync_api import Error, sync_playwright
+
+    state: Optional[dict] = None
+    try:
+        with sync_playwright() as driver:
+            browser = driver.chromium.launch(headless=False)
+            try:
+                context = browser.new_context()
+                context.new_page().goto(url)
+                deadline = time.monotonic() + wait_seconds
+                while time.monotonic() < deadline:
+                    try:
+                        state = context.storage_state()
+                    except Error:
+                        break
+                    if not context.pages:
+                        break
+                    time.sleep(1)
+            finally:
+                try:
+                    browser.close()
+                except Error:
+                    pass
+    except Error as exc:
+        raise BackendError(f"The browser could not be started or could not open {url}: {exc}") from None
+    if state is None:
+        raise BackendError("The browser window closed before any state could be read - nothing was stored.")
+    return state
+
+
+def render(
+    url: str, user_agent: str, state: Optional[dict] = None,
+    timeout: float = web_capture.TIMEOUT_SECONDS, max_bytes: int = web_capture.MAX_BYTES,
+) -> web_capture.Response:
+    """Load `url` in a headless browser - with `state`, the user's session,
+    when given - and return the page as the browser drew it: the serialised
+    DOM after the network went quiet, not the bytes the server sent."""
+    require(f"{url} is declared `render` by its site handler, so the fetch")
+    _environment()
+    from playwright.sync_api import Error, sync_playwright
+
+    web_capture.check_url(url)
+    try:
+        with sync_playwright() as driver:
+            browser = driver.chromium.launch(headless=True)
+            try:
+                context = browser.new_context(user_agent=user_agent, storage_state=state)
+                page = context.new_page()
+                response = page.goto(url, wait_until="networkidle", timeout=timeout * 1000)
+                body = page.content().encode("utf-8")
+                final_url = page.url
+                status = response.status if response is not None else 0
+            finally:
+                browser.close()
+    except Error as exc:
+        raise BackendError(f"The browser could not load {url}: {exc}") from None
+    web_capture.check_url(final_url)
+    if status >= 400:
+        raise web_capture.HttpError(f"{url} answered HTTP {status}.", status)
+    if len(body) > max_bytes:
+        raise BackendError(f"{url} is larger than {max_bytes // (1024 * 1024)} MiB as drawn by the browser.")
+    return web_capture.Response(url, final_url, status, "text/html; charset=utf-8", body)

tools/chemenu/cli_contract.py

@@ -287,7 +287,7 @@ GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = (
         "sources coverage", "sources trace", "sources rebuild-index",
     )),
     ("Raw material and uploads", (
-        "raw fetch", "raw derive", "raw capture", "raw status", "raw pending", "raw accept",
+        "raw fetch", "raw login", "raw derive", "raw capture", "raw status", "raw pending", "raw accept",
         "upload list", "upload show", "upload accept", "upload reject",
     )),
     ("Git", (

tools/chemenu/commands/doctor.py

@@ -804,6 +804,32 @@ def check_site_handlers() -> Check:
     return Check("site-handlers", "OK", f"{len(instance)} handler(s) of this instance: {detail}")
 
 
+def check_browser() -> Check:
+    """The optional browser component: OK whether or not it is installed, a
+    WARN only when this instance has something that needs it - a host routed
+    through the session (`raw login`) or a handler that declares `render`.
+    Never a FAIL: every other command runs without it."""
+    from chemenu import browser, settings, sites
+
+    why = browser.missing()
+    if why is None:
+        return Check("browser", "OK", f"available - Playwright in the venv, its browser in {rel_path(browser.browsers_path())}")
+    try:
+        routes = settings.load(only=("fetch",)).section("fetch")["routes"]
+    except ChemenuError:
+        routes = {}  # a broken section is reported under settings-*
+    sessions = sorted(host for host, via in routes.items() if via == config_shapes.SESSION_ROUTE)
+    instance, _ = sites.read_instance()
+    rendered = sorted(site.name for site in sites.catalogue(instance) if site.render)
+    needs = ([f"`raw login` for {', '.join(sessions)}"] if sessions else []) + (
+        [f"the handler(s) {', '.join(rendered)}, which declare `render`"] if rendered else [])
+    if needs:
+        return Check("browser", "WARN", f"not available ({why}) - needed by {' and '.join(needs)}",
+                     f"The user installs it: {browser.PREFLIGHT_WITH}")
+    return Check("browser", "OK", f"not installed - optional, needed only by `raw login` and handlers "
+                 f"that declare `render`; the user installs it with {browser.PREFLIGHT_WITH}")
+
+
 def check_session_id() -> Check:
     """Three-valued, not two: an explicit `WIKITOOL_SESSION_ID` and a
     recognised harness variable (see `chemenu.session.HARNESS_ENV_VARS`) both
@@ -927,6 +953,7 @@ def run_doctor() -> list[Check]:
         check_tasks_provider(),
         *check_settings(),
         check_site_handlers(),
+        check_browser(),
         check_generated_files(),
         check_session_id(),
         check_telemetry(),
@@ -1007,6 +1034,11 @@ def run_doctor() -> list[Check]:
         "schema and the rules the schema cannot state; `FAIL` for a file that cannot be used, "
         "naming the file and the reason, and `OK` listing the instance's handlers, each one that "
         "replaces a stack handler (the same host and path) named as such.",
+        "Browser component (`browser`): whether the optional Playwright component and its browser "
+        "are installed in the venv - `OK` either way, never a `FAIL`; a `WARN` only when it is "
+        "missing and the instance needs it: a host `fetch.routes` sends through the session "
+        "(`raw login`), or a site handler that declares `render`. The user installs it with "
+        "`tools/preflight.sh --with browser`.",
         "Exits 1 only on a `FAIL`; a missing remote, session id or `VERSION` is a `WARN`, not a "
         "fault.",
         "Read-only and exempt from the Iteration Budget Gate.",

tools/chemenu/commands/raw/__init__.py

@@ -7,13 +7,14 @@ subcommands are listed in `-h` and in the generated command reference.
 """
 import typer
 
-from chemenu.commands.raw import accept, pending, fetch, derive, capture
+from chemenu.commands.raw import accept, pending, fetch, login, derive, capture
 
 app = typer.Typer(help="Bring raw material into incoming/, and promote it from there into raw/.")
 
 app.command("accept")(accept.raw_accept_command)
 app.command("pending")(pending.raw_pending_command)
 app.command("fetch")(fetch.raw_fetch_command)
+app.command("login")(login.raw_login_command)
 app.command("derive")(derive.raw_derive_command)
 app.command("capture")(capture.raw_capture_command)
 app.command("status")(capture.raw_status_command)

tools/chemenu/commands/raw/fetch.py

@@ -20,8 +20,16 @@ from typing import Annotated, Optional
 import typer
 from rich.markup import escape
 
-from chemenu import cli_contract, config, sites, web_capture
-from chemenu.commands._util import fail, path_budget_problem_for, rel_path, success
+from chemenu import browser, cli_contract, config, fetch_routes, sites, web_capture
+from chemenu.commands._util import (
+    clearance_on_settings,
+    fail,
+    load_settings,
+    needs_clearance,
+    path_budget_problem_for,
+    rel_path,
+    success,
+)
 from chemenu.errors import ChemenuError
 from chemenu.version import VersionError, read_version
 from chemenu.commands.raw._common import (
@@ -102,6 +110,50 @@ def _plan(url: str, *, canonical: bool, preferences: Optional[sites.Preferences]
         fail(escape(str(exc)))
 
 
+def _fetch_section():
+    """The `fetch` settings section - which hosts go another way than the
+    built-in anonymous fetch."""
+    with clearance_on_settings():
+        return load_settings(only=("fetch",)).section("fetch")
+
+
+def _route(plan: sites.Plan, preferences: sites.Preferences, section) -> fetch_routes.Route:
+    try:
+        return fetch_routes.route_for(plan, preferences, section)
+    except ChemenuError as exc:
+        fail(escape(str(exc)))
+
+
+def _get(route: fetch_routes.Route, url: str, agent: str, context: str = "") -> web_capture.Response:
+    """Fetch `url` along `route`: exit 42 when only the user can help (no
+    session, no browser component), exit 1 for everything else."""
+    try:
+        return fetch_routes.get(route, url, agent)
+    except (fetch_routes.SessionNeeded, browser.BrowserMissing) as exc:
+        needs_clearance(escape(str(exc)))
+    except ChemenuError as exc:
+        fail(escape(f"{exc}{context}"))
+
+
+def _signed_in(route: fetch_routes.Route, response: web_capture.Response) -> None:
+    try:
+        fetch_routes.check_signed_in(route, response)
+    except fetch_routes.SessionNeeded as exc:
+        needs_clearance(escape(str(exc)))
+
+
+def _route_entries(route: fetch_routes.Route) -> dict:
+    """What `_scope.json` says about a route other than the built-in fetch."""
+    found: dict = {}
+    if route.kind == fetch_routes.SESSION:
+        found["session"] = route.host
+    if route.kind == fetch_routes.FETCHER:
+        found["fetcher"] = route.fetcher
+    if route.render:
+        found["rendered"] = True
+    return found
+
+
 def _echo_checks(plan: sites.Plan, derivation: web_capture.Derivation, subpages: int = 0) -> None:
     for line in plan.checks(derivation.decoded.text, subpages):
         typer.echo(f"CHECK {line}")
@@ -202,22 +254,21 @@ def _fetch_scope(url: str, name: Optional[str]) -> None:
         ))
     refuse_taken()
 
+    section = _fetch_section()
+    route = _route(plan, preferences, section)
     agent = _user_agent()
-    try:
-        index = web_capture.fetch(
-            plan.url, agent, timeout=web_capture.TIMEOUT_SECONDS, max_bytes=web_capture.MAX_BYTES
-        )
-    except ChemenuError as exc:
-        fail(escape(str(exc)))
+    index = _get(route, plan.url, agent)
     index_retrieved = web_capture.utc_now()
     if index.media_type not in web_capture.HTML_TYPES:
         fail(escape(
             f"{plan.url} answered {index.media_type or 'no content type'}, not an HTML page - "
             "--scope bundles HTML pages. Nothing was written."
         ))
+    _signed_in(route, index)
     index_derivation = web_capture.derive_document(
         index.body, index.final_url, f"{stem}.html", response=index,
         drop=plan.drop, site=plan.label, requested_url=plan.requested_url, derived_at=index_retrieved,
+        fetched_by=route.fetched_by, session=route.session_label, rendered=route.render,
     )
     urls = plan.subpages(index_derivation.decoded.text, index.final_url)
     if not urls:
@@ -261,12 +312,9 @@ def _fetch_scope(url: str, name: Optional[str]) -> None:
         for position, (sub_url, sub_name) in enumerate(zip(urls, names), start=1):
             if preferences.pause_seconds:
                 time.sleep(preferences.pause_seconds)
-            try:
-                page = web_capture.fetch(
-                    sub_url, agent, timeout=web_capture.TIMEOUT_SECONDS, max_bytes=web_capture.MAX_BYTES
-                )
-            except ChemenuError as exc:
-                fail(escape(f"{exc} (subpage {position} of {len(urls)}) Nothing was written."))
+            sub_plan = _plan(sub_url, canonical=False, preferences=preferences)
+            sub_route = _route(sub_plan, preferences, section)
+            page = _get(sub_route, sub_url, agent, f" (subpage {position} of {len(urls)}) Nothing was written.")
             retrieved = web_capture.utc_now()
             if urllib.parse.urlsplit(page.final_url).netloc.lower() != host:
                 fail(escape(
@@ -278,10 +326,11 @@ def _fetch_scope(url: str, name: Optional[str]) -> None:
                     f"{sub_url} answered {page.media_type or 'no content type'}, not an HTML page - "
                     "--scope bundles HTML pages. Nothing was written."
                 ))
-            sub_plan = _plan(sub_url, canonical=False, preferences=preferences)
+            _signed_in(sub_route, page)
             derivation = web_capture.derive_document(
                 page.body, page.final_url, f"{sub_name}.html", response=page,
                 drop=sub_plan.drop, site=sub_plan.label, derived_at=retrieved,
+                fetched_by=sub_route.fetched_by, session=sub_route.session_label, rendered=sub_route.render,
             )
             (bundle / f"{sub_name}.html").write_bytes(page.body)
             (bundle / f"{sub_name}.md").write_bytes(derivation.markdown)
@@ -293,6 +342,7 @@ def _fetch_scope(url: str, name: Optional[str]) -> None:
             "url": plan.url,
             **({"requested_url": plan.requested_url} if plan.requested_url else {}),
             "retrieved": index_retrieved,
+            **_route_entries(route),
             "pages": entries,
         }
         (bundle / SCOPE_MANIFEST).write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8", newline="\n")
@@ -317,7 +367,8 @@ def _fetch_scope(url: str, name: Optional[str]) -> None:
     path="raw fetch",
     summary="Capture a web page the user names into `incoming/`: the HTML as received plus a "
     "derived text, for `raw accept` to promote. Pages of a known website are fetched in the "
-    "website's standard form and cleaned of its page furniture.",
+    "website's standard form and cleaned of its page furniture; a host the instance routes so is "
+    "fetched with the user's own session or through an external fetcher.",
     synopsis=(
         cli_contract.Variant(
             usage="raw fetch <url> [--name <stem>]",
@@ -394,8 +445,27 @@ def _fetch_scope(url: str, name: Optional[str]) -> None:
         "other non-HTML response (PDF, image, ...) as received under the extension of its "
         "content type. Neither gets a derived file or a header.",
         "Only `http`/`https`, on every redirect hop too. 30 s for the whole transfer, 25 MiB at "
-        "most. No cookies, no JavaScript: a derived text under 200 characters is written, with "
-        "a warning to read it before accepting.",
+        "most. No cookies and no JavaScript unless the host is routed or its handler says so "
+        "(below): a derived text under 200 characters is written, with a warning to read it "
+        "before accepting.",
+        "Fetch routes: the `fetch` section of the settings sends a host's pages another way. "
+        "`session` fetches them with the cookies of the session the user stored with `raw login`, "
+        "still with the built-in fetch and the bytes as received; the header adds `session: "
+        "<host>` and never the session's content. The host's site handler names how a page shows "
+        "it arrived signed in: no stored session, or a page without that sign (an expired "
+        "session, a teaser), stops with exit 42 and writes nothing. Any other route names an "
+        "external fetcher of this checkout: run without a shell with the URL as its last "
+        "argument (120 s, 25 MiB), its stdout is the `.html` byte for byte, and the header says "
+        "`fetched_by: wikitool raw fetch (fetcher: <name>)` and carries no `http_status`. A "
+        "fetcher that fails is never replaced by the built-in fetch. A host no route names is "
+        "fetched as before.",
+        "Rendered pages: a site handler that declares `render` has its pages fetched through a "
+        "headless browser - with the session, when the host has one - and the `.html` is the page "
+        "as the browser drew it, not the bytes the server sent; the header says `rendered: true`. "
+        "No page is rendered without that declaration, and an external fetcher's page never is. "
+        "It needs the optional browser component; without it the fetch stops with exit 42. "
+        "`--scope` fetches every page of the bundle the same way, and `_scope.json` names the "
+        "`session`, `fetcher` or `rendered` it used.",
         "`--html` derives the `.md` beside a page saved to `incoming/` from a logged-in browser "
         "- the way past a paywall or a script-rendered page. Its header carries `derived` (when "
         "the text was derived) instead of `retrieved`, `final_url`, `http_status` and "
@@ -430,6 +500,35 @@ def _fetch_scope(url: str, name: Optional[str]) -> None:
             "with the user; for a paywall or login, save the page in a browser and use "
             "`--html`. An unreachable host or a timeout may be retried once",
         ),
+        cli_contract.Failure(
+            cause="The host is fetched with the user's session and none is stored, or the page "
+            "arrived without the sign of being signed in that its site handler names (the session "
+            "expired)",
+            reaction="Nothing was written. Show the user the output and stop: they sign in with "
+            "`tools/wikitool raw login <host>`, never you. Retry once they say it is done",
+            code=42,
+        ),
+        cli_contract.Failure(
+            cause="The site handler declares `render` and the optional browser component is not "
+            "installed",
+            reaction="Nothing was written. Show the user the output and stop: they install it with "
+            "`tools/preflight.sh --with browser`. Retry once they say it is done",
+            code=42,
+        ),
+        cli_contract.Failure(
+            cause="The `fetch` section is malformed, routes the host through the session though "
+            "no site handler of the host names a login page, or names a fetcher this checkout does "
+            "not define",
+            reaction="Nothing was fetched or written. Fix the route with `tools/wikitool config "
+            "set`, or have the user add the fetcher to `.wikitool-local.json` by hand - it names a "
+            "program, so `config set` does not write it - then retry once",
+        ),
+        cli_contract.Failure(
+            cause="The external fetcher could not be started, timed out, ended with an exit other "
+            "than 0, printed nothing or more than 25 MiB",
+            reaction="Nothing was written, and the built-in fetch was not tried. The message carries "
+            "the fetcher's stderr - report it to the user; do not fetch the page another way",
+        ),
         cli_contract.Failure(
             label="raw fetch --scope",
             cause="No site handler offers subpages for the URL, the page offers none or more than "
@@ -462,7 +561,9 @@ def _fetch_scope(url: str, name: Optional[str]) -> None:
     see_also=(
         "`raw/CONTRACT.md` \"Getting a URL in: `raw fetch`\" - the rules and why",
         "`sites/CONTRACT.md` - the instance's own handler files",
-        "`docs/reference/configuration.md` - the `sites` section, which tunes the site handlers",
+        "`docs/reference/configuration.md` - the `sites` section, which tunes the site handlers, "
+        "and the `fetch` section, which routes a host",
+        "`wikitool raw login` - the user signs in once for a host routed through the session",
         "`wikitool raw accept` - promotes the written files into `raw/`",
         "`instructions/wiki-ingest/SKILL.md` - where a URL to ingest starts",
     ),
@@ -508,32 +609,31 @@ def raw_fetch_command(
     if scope:
         _fetch_scope(url, name)
         return
-    plan = _plan(url, canonical=True)
+    preferences = _preferences()
+    plan = _plan(url, canonical=True, preferences=preferences)
     stem = name or web_capture.stem_for(plan.url)
-
-    try:
-        response = web_capture.fetch(
-            plan.url, _user_agent(), timeout=web_capture.TIMEOUT_SECONDS, max_bytes=web_capture.MAX_BYTES
-        )
-    except ChemenuError as exc:
-        fail(escape(str(exc)))
+    route = _route(plan, preferences, _fetch_section())
+    response = _get(route, plan.url, _user_agent())
 
     incoming = _incoming_dir()
     kind = response.media_type
     if kind in web_capture.HTML_TYPES:
         html_path, md_path = incoming / f"{stem}.html", incoming / f"{stem}.md"
         _refuse_existing([html_path, md_path])
+        _signed_in(route, response)
         derivation = web_capture.derive_document(
             response.body, response.final_url, html_path.name, response=response,
             drop=plan.drop, site=plan.label, requested_url=plan.requested_url,
+            fetched_by=route.fetched_by, session=route.session_label, rendered=route.render,
         )
         _write_exclusive([(html_path, response.body), (md_path, derivation.markdown)])
         _warn(derivation, md_path)
         _echo_checks(plan, derivation, len(plan.subpages(derivation.decoded.text, response.final_url)))
+        anonymous = route.kind == fetch_routes.PLAIN and not route.render
         success(escape(
             f"Fetched {_fetched(plan)} to {rel_path(html_path)}, {rel_path(md_path)}\n"
-            + _next_lines([html_path, md_path], plan.url) + "\n"
-            + _TEASER_HINT.format(url=plan.url)
+            + _next_lines([html_path, md_path], plan.url)
+            + ("\n" + _TEASER_HINT.format(url=plan.url) if anonymous else "")
         ))
         return
 

tools/chemenu/commands/raw/login.py

@@ -0,0 +1,141 @@
+"""`wikitool raw login <host>` - the user signs in to a website once, in a visible
+browser window, and the session it leaves is stored as a secret outside the
+working tree for `raw fetch` to use. Only a human runs it: it waits for someone
+at the window. The browser part is `chemenu.browser`, the storage
+`chemenu.web_session`; this adapter checks the order of things and says what
+happened.
+"""
+from __future__ import annotations
+
+import typer
+from rich.markup import escape
+
+from chemenu import browser, cli_contract, config, sites, web_session
+from chemenu.commands._util import clearance_on_settings, fail, load_settings, needs_clearance, success
+from chemenu.config_shapes import SESSION_ROUTE
+from chemenu.errors import ChemenuError
+from chemenu.fetch_routes import host_of
+
+
+def _host(value: str) -> str:
+    host = value.strip().lower()
+    if "://" in host:
+        host = host_of(host)
+    if not host or any(c in host for c in "/:?#@ "):
+        fail(escape(f"{value!r} is not a host name - pass the host alone, such as www.example.org."))
+    return host
+
+
+@cli_contract.record(cli_contract.CommandRecord(
+    path="raw login",
+    summary="Let the user sign in to a website once, in a visible browser window, and store the "
+    "session it leaves as a secret outside the working tree, for `raw fetch` to fetch that "
+    "website's pages with. Only the user runs it.",
+    synopsis=(cli_contract.Variant(usage="raw login <host>"),),
+    properties=cli_contract.Properties(
+        effect=cli_contract.Effect.WRITE,
+        idempotent=cli_contract.Idempotent.YES,
+        atomic="Yes - the secrets file is replaced in one rename, with every other entry kept",
+        budget=cli_contract.Budget.COUNTED,
+        network=cli_contract.Network.YES,
+    ),
+    notes=(
+        "Opens the login page the host's site handler names (its `session` block) in a visible "
+        "browser window and waits until the user has signed in - captcha and second factor "
+        "included - and closed the window, at most 15 minutes.",
+        "Stores only the browser state the window leaves (cookies and storage), never a "
+        "password, under `session_<host>` (every run of other characters as one `_`) in the "
+        "secrets file of the instance, outside the working tree and readable by its owner only. "
+        "`WIKITOOL_SECRET_SESSION_<HOST>` answers before the file, as for any named secret. The "
+        "session is never written to 1Password.",
+        "Needs the host routed through the session first (`fetch.routes`, set with `wikitool "
+        "config set`), which also gives the instance the id its secrets file is found by.",
+        "Needs the optional browser component, which the user installs with the preflight's "
+        "`--with browser` option.",
+        "Prints no part of the session. Signing in again replaces the stored session.",
+    ),
+    failures=(
+        cli_contract.Failure(
+            cause="The host is not routed through the session in `fetch.routes`, no enabled site "
+            "handler of the host names a login page, or the `fetch` settings are malformed",
+            reaction="Nothing was stored. Route the host (`tools/wikitool config set fetch.routes "
+            "'{\"<host>\": \"session\"}'`), add a `session` block to its handler (`sites/CONTRACT.md`), "
+            "or fix the section, then retry once",
+        ),
+        cli_contract.Failure(
+            cause="The browser could not be started, or the window closed before any state could "
+            "be read",
+            reaction="Nothing was stored. Read the browser's message - on Linux it may name system "
+            "libraries the browser needs, which whoever administers the machine installs - then retry once",
+        ),
+        cli_contract.Failure(
+            cause="The optional browser component is not installed",
+            reaction="Nothing was stored. The user installs it with `tools/preflight.sh --with "
+            "browser`, then runs this command again",
+            code=42,
+        ),
+    ),
+    examples=("tools/wikitool raw login www.heise.de",),
+    never=(
+        "Never run this command as an agent - it waits for a human at a browser window. When "
+        "`raw fetch` asks for a sign-in, show the user its output and let them run it.",
+        "Never read, print or copy the stored session.",
+    ),
+    see_also=(
+        "`wikitool raw fetch` - fetches the pages of a routed host with the stored session",
+        "`raw/CONTRACT.md` \"Getting a URL in: `raw fetch`\" - fetching behind a login",
+        "`sites/CONTRACT.md` - a handler's `session` block",
+        "`docs/reference/configuration.md` - the `fetch` section",
+    ),
+))
+def raw_login_command(
+    host: str = typer.Argument(..., help="The website's host, such as www.example.org"),
+):
+    """Let the user sign in to a website in a browser window and store the session.
+    Only a human runs this. See raw/CONTRACT.md "Getting a URL in: `raw fetch`"."""
+    host = _host(host)
+    with clearance_on_settings():
+        loaded = load_settings(only=("fetch",))
+    section = loaded.section("fetch")
+    route = section["routes"].get(host)
+    if route != SESSION_ROUTE:
+        fail(escape(
+            f"{host} is not fetched with a session - `fetch.routes` {'sends it to ' + repr(route) if route else 'does not list it'}. "
+            f"Route it first: tools/wikitool config set fetch.routes '{{\"{host}\": \"{SESSION_ROUTE}\"}}' "
+            "(keep the hosts already listed there)."
+        ))
+    try:
+        preferences = sites.read_preferences()
+    except ChemenuError as exc:
+        fail(escape(str(exc)))
+    site = sites.login_site(host, preferences.disabled, preferences.instance)
+    if site is None or site.session is None:
+        fail(escape(
+            f"No enabled site handler of {host} names a login page - add a `session` block to a handler "
+            f"of {host} (sites/CONTRACT.md)."
+        ))
+    if not loaded.instance_id:
+        fail(escape(
+            f"{config.INSTANCE_SETTINGS_FILENAME} has no instance_id, so there is no secrets file to store "
+            "the session in - set the route with `tools/wikitool config set`, which writes one."
+        ))
+    try:
+        state = browser.login(site.session.login)
+    except browser.BrowserMissing as exc:
+        needs_clearance(escape(str(exc)))
+    except ChemenuError as exc:
+        fail(escape(f"{exc} Nothing was stored."))
+    try:
+        path = web_session.store(host, loaded.instance_id, state)
+    except (ChemenuError, OSError) as exc:
+        fail(escape(f"The session could not be stored: {exc}"))
+    cookies = len(state.get("cookies", [])) if isinstance(state, dict) else 0
+    shadow = web_session.overridden(host)
+    if shadow:
+        typer.echo(f"WARN {shadow} is set and answers before the secrets file - unset it, or the "
+                   "session just stored is not used.")
+    success(escape(
+        f"Signed in to {host}: the session ({cookies} cookie(s)) is stored as "
+        f"{web_session.secret_name(host)} in {path}, outside the working tree. "
+        f"`raw fetch` uses it for {host} until it expires; then it asks for this command again."
+    ))

tools/chemenu/config_shapes.py

@@ -244,6 +244,38 @@ def site_notes_problem(value: list[dict]) -> Optional[str]:
     return None
 
 
+SESSION_ROUTE = "session"
+_HOST = re.compile(r"[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*")
+_FETCHER_NAME = re.compile(r"[a-z0-9]+(-[a-z0-9]+)*")
+
+
+def object_value(value: Any) -> dict:
+    if not isinstance(value, dict):
+        raise TypeError("not an object")
+    return dict(value)
+
+
+def fetch_routes_problem(value: dict) -> Optional[str]:
+    for host, route in value.items():
+        if not _HOST.fullmatch(host):
+            return f"has the key {host!r}, which is not a lower-case host name"
+        if not isinstance(route, str) or not _FETCHER_NAME.fullmatch(route):
+            return (f"maps {host} to {route!r} - a route is {SESSION_ROUTE!r} or the name of a fetcher "
+                    "(lower-case letters, digits and hyphens)")
+    return None
+
+
+def fetchers_problem(value: dict) -> Optional[str]:
+    for name, command in value.items():
+        if name == SESSION_ROUTE or not _FETCHER_NAME.fullmatch(name):
+            return (f"has the name {name!r} - a fetcher's name is lower-case letters, digits and "
+                    f"hyphens, and not {SESSION_ROUTE!r}")
+        if (not isinstance(command, list) or not command
+                or not all(isinstance(part, str) and part for part in command)):
+            return f"{name} must be a non-empty array of non-empty strings: the program, then its arguments"
+    return None
+
+
 def _positive(value: int) -> Optional[str]:
     return None if value > 0 else "must be positive"
 
@@ -510,11 +542,45 @@ SITES = Shape(
     "broken section is never read as \"no handlers\".",
 )
 
+FETCH = Shape(
+    key="fetch",
+    title="Fetch routes",
+    purpose="Sends the pages of a host another way than the built-in anonymous fetch of `raw "
+    "fetch`: with the session the user signed in with through `raw login`, or through an external "
+    "fetcher program of this checkout.",
+    fields=(
+        Field(
+            "routes", "object", f"Host (exact, lower case) to route: `\"{SESSION_ROUTE}\"` fetches "
+            "with the user's session - the host needs a site handler that names its login page - "
+            "and any other value names a fetcher of `fetchers`. A host not listed is fetched as "
+            "before.",
+            default={}, json_type="object", coerce=object_value, check=fetch_routes_problem,
+            tier=INSTANCE,
+        ),
+        Field(
+            "fetchers", "object", "Fetcher name to command: an array of the program and its "
+            "arguments, run without a shell with the URL appended. Exit 0 and the page's HTML on "
+            "stdout is a fetch; anything else stops `raw fetch` with exit 1 and the program's "
+            "stderr.",
+            default={}, json_type="object", coerce=object_value, check=fetchers_problem,
+            tier=CHECKOUT, names_command=True,
+        ),
+    ),
+    examples=({
+        "routes": {"www.heise.de": SESSION_ROUTE, "intranet.example.org": "homelab"},
+        "fetchers": {"homelab": ["/usr/local/bin/fetch-page", "--html"]},
+    },),
+    absent="Every page is fetched by the built-in anonymous fetch, as before.",
+    malformed="An error when `raw fetch` or `raw login` reads it: the command stops with exit 1 "
+    "before fetching anything. A route to a fetcher this checkout does not define stops a fetch "
+    "of that host the same way.",
+)
+
 # The sections of the tiered settings files, `wikitool.json` and
 # `.wikitool-local.json` (`chemenu.settings`), read at call time so a test can
 # stand in a section of its own.
 SETTINGS_SECTIONS: tuple[Shape, ...] = (
-    TELEMETRY, REMOTES, UPLOAD, TASKS, SUPERPRODUCTIVITY, CALDAV, SITES,
+    TELEMETRY, REMOTES, UPLOAD, TASKS, SUPERPRODUCTIVITY, CALDAV, SITES, FETCH,
 )
 
 # The sections `WIKITOOL_TASKS_CONFIG` replaces as a whole: a profile names a

tools/chemenu/fetch_routes.py

@@ -0,0 +1,179 @@
+"""Which way a page is fetched: the built-in anonymous fetch, the user's own
+session, or an external fetcher - and the fetch itself.
+
+The instance decides per host (`fetch.routes`, tier `instance`):
+
+- not listed: the built-in fetch, exactly as before - no cookie, no browser;
+- `session`: the built-in fetch with the cookies of the session the user signed
+  in with through `raw login` (`chemenu.web_session`). The host's site handler
+  names the login page and how a page shows that it arrived signed in; a page
+  that did not, or no stored session at all, is `SessionNeeded` - exit 42,
+  because only the user can sign in again. A teaser is never kept in silence;
+- any other value: an external fetcher of this checkout (`fetch.fetchers`, tier
+  `checkout`, never `instance` - it names a program to run). It is run without
+  a shell with the URL appended; exit 0 and HTML on stdout is the page,
+  anything else is an error, and the built-in fetch is never tried instead.
+
+A site handler that declares `render` sends a fetch through a headless browser
+instead of the built-in fetch, with the session when the route has one
+(`chemenu.browser`); an external fetcher is its own fetch and is never rendered.
+
+No `typer`, no `rich`: errors are `ChemenuError`s, and the CLI decides what each
+one means for the exit code.
+"""
+from __future__ import annotations
+
+import subprocess
+import urllib.parse
+from dataclasses import dataclass
+from typing import Optional
+
+from chemenu import browser, config, sites, web_capture, web_session
+from chemenu.config_shapes import SESSION_ROUTE
+from chemenu.errors import BackendError, ChemenuError, ValidationError
+
+PLAIN = "plain"
+SESSION = "session"
+FETCHER = "fetcher"
+FETCHER_TIMEOUT_SECONDS = 120
+_STDERR_TAIL_CHARS = 2000
+
+
+class SessionNeeded(ChemenuError):
+    """No usable session for the host: none stored, or the page arrived
+    without the sign of being signed in. Only the user can sign in, so the CLI
+    renders this as exit 42."""
+
+
+@dataclass(frozen=True)
+class Route:
+    """How one page is fetched. `session` is the handler's description of
+    the login, for a `SESSION` route; `command` the program and arguments of a
+    `FETCHER` route; `render` whether the page goes through a browser."""
+
+    host: str
+    kind: str = PLAIN
+    fetcher: str = ""
+    command: tuple[str, ...] = ()
+    session: Optional[sites.Session] = None
+    render: bool = False
+    instance_id: Optional[str] = None
+
+    @property
+    def fetched_by(self) -> str:
+        if self.kind == FETCHER:
+            return f"wikitool raw fetch (fetcher: {self.fetcher})"
+        return "wikitool raw fetch"
+
+    @property
+    def session_label(self) -> str:
+        return self.host if self.kind == SESSION else ""
+
+
+def host_of(url: str) -> str:
+    return (urllib.parse.urlsplit(url).hostname or "").lower()
+
+
+def route_for(plan: sites.Plan, preferences: sites.Preferences, section) -> Route:
+    """The route for the page `plan` describes, from the `fetch` settings
+    `section` (`chemenu.settings.Section`). Raises `ValidationError` for a route
+    that cannot work: `session` for a host no handler names a login page for,
+    or a fetcher this checkout does not define."""
+    host = host_of(plan.url)
+    render = bool(plan.site and plan.site.render)
+    via = section["routes"].get(host)
+    if via is None:
+        return Route(host, render=render)
+    if via == SESSION_ROUTE:
+        login = sites.login_site(host, preferences.disabled, preferences.instance)
+        if login is None:
+            raise ValidationError(
+                f"fetch.routes sends {host} through the user's session, and no enabled site handler of "
+                f"that host names a login page - add a `session` block to a handler of {host} "
+                "(sites/CONTRACT.md), or remove the route."
+            )
+        own = plan.site.session if plan.site and plan.site.session else login.session
+        return Route(host, SESSION, session=own, render=render, instance_id=section.instance_id)
+    command = section["fetchers"].get(via)
+    if command is None:
+        raise ValidationError(
+            f"fetch.routes sends {host} to the fetcher {via!r}, which this checkout does not define - "
+            f"the user adds `fetch.fetchers.{via}` to {config.LOCAL_SETTINGS_FILENAME} by hand "
+            "(it names a program to run, so `config set` does not write it)."
+        )
+    return Route(host, FETCHER, fetcher=via, command=tuple(command))
+
+
+def _login_hint(host: str) -> str:
+    return (f"Ask the user to run `tools/wikitool raw login {host}` - it opens a browser window for "
+            "them to sign in; never run it yourself. Nothing was written.")
+
+
+def get(route: Route, url: str, user_agent: str) -> web_capture.Response:
+    """Fetch `url` the way `route` says. Raises `SessionNeeded` when no
+    session is stored, `browser.BrowserMissing` when a rendered fetch has no
+    browser, and `BackendError`/`ValidationError` for everything else."""
+    if route.kind == FETCHER:
+        return run_fetcher(route, url)
+    state = None
+    if route.kind == SESSION:
+        state = web_session.load(route.host, route.instance_id)
+        if state is None:
+            raise SessionNeeded(f"{route.host} is fetched with the user's session, and none is stored. "
+                                + _login_hint(route.host))
+    if route.render:
+        return browser.render(url, user_agent, state)
+    # The built-in fetch exactly as before when there is no session to send.
+    extra = {"cookies": web_session.cookie_jar(state)} if state is not None else {}
+    return web_capture.fetch(
+        url, user_agent, timeout=web_capture.TIMEOUT_SECONDS, max_bytes=web_capture.MAX_BYTES, **extra
+    )
+
+
+def check_signed_in(route: Route, response: web_capture.Response) -> None:
+    """Raise `SessionNeeded` when a page fetched with the session did not
+    arrive signed in - the session expired, or the website ended it."""
+    if route.kind != SESSION or route.session is None or response.media_type not in web_capture.HTML_TYPES:
+        return
+    text = web_capture.decode(response.body, response.content_type).text
+    if not route.session.is_signed_in(text):
+        raise SessionNeeded(
+            f"{response.final_url} arrived without a valid session - the page does not show the sign of "
+            f"being signed in that the site handler names, so the session for {route.host} has expired "
+            "or was ended. " + _login_hint(route.host)
+        )
+
+
+def _tail(stderr: bytes) -> str:
+    text = stderr.decode("utf-8", errors="replace").strip()
+    if len(text) > _STDERR_TAIL_CHARS:
+        text = "..." + text[-_STDERR_TAIL_CHARS:]
+    return text or "(nothing on stderr)"
+
+
+def run_fetcher(route: Route, url: str, timeout: float = FETCHER_TIMEOUT_SECONDS,
+                max_bytes: int = web_capture.MAX_BYTES) -> web_capture.Response:
+    """Run the external fetcher of `route` for `url`: no shell, the URL as the
+    last argument, stdin closed. Its stdout is the page, byte for byte."""
+    web_capture.check_url(url)
+    where = f"the fetcher {route.fetcher!r}"
+    try:
+        result = subprocess.run(
+            [*route.command, url], capture_output=True, stdin=subprocess.DEVNULL, timeout=timeout, check=False,
+        )
+    except FileNotFoundError:
+        raise BackendError(f"{where} could not be started: {route.command[0]} does not exist. Nothing was written.") from None
+    except subprocess.TimeoutExpired:
+        raise BackendError(f"{where} did not finish within {timeout:g} s for {url}. Nothing was written.") from None
+    except OSError as exc:
+        raise BackendError(f"{where} could not be started ({exc}). Nothing was written.") from None
+    if result.returncode != 0:
+        raise BackendError(
+            f"{where} ended with exit {result.returncode} for {url} - nothing was written, and the built-in "
+            f"fetch was not tried instead. Its stderr:\n{_tail(result.stderr)}"
+        )
+    if not result.stdout:
+        raise BackendError(f"{where} ended with exit 0 for {url} but printed nothing. Nothing was written.")
+    if len(result.stdout) > max_bytes:
+        raise BackendError(f"{where} printed more than {max_bytes // (1024 * 1024)} MiB for {url}. Nothing was written.")
+    return web_capture.Response(url, url, 0, "text/html", result.stdout)

tools/chemenu/reference_doc.py

@@ -69,6 +69,7 @@ CONFIG_REGIONS = (
     ("reference-config-caldav", "CALDAV"),
     ("reference-config-upload", "UPLOAD"),
     ("reference-config-sites", "SITES"),
+    ("reference-config-fetch", "FETCH"),
 )
 
 _SLUG_RE = re.compile(r"[^a-z0-9]+")

tools/chemenu/settings.py

@@ -558,6 +558,21 @@ def read_secrets_file(path: Path) -> Optional[dict]:
     return data
 
 
+def write_secret(instance_id: str, name: str, value: str) -> Path:
+    """Put `value` under `name` into the secrets file of `instance_id`, keeping
+    every other entry, and return the file's path. The file is replaced in one
+    rename and created readable by its owner only (`write_file` goes through
+    `mkstemp`). Used by what writes a secret the user produced but never typed -
+    `raw login`'s session; `config set` refuses a secret value outright."""
+    if not _NAME_RE.match(name):
+        raise ValidationError(f"{name!r} is not a secret name: lower-case letters, digits and single underscores")
+    path = config.secrets_file(instance_id)
+    data = read_secrets_file(path) or {}
+    data[name] = value
+    write_file(path, data)
+    return path
+
+
 def _resolve_named(ref: Ref, key: str, instance_id: Optional[str]) -> str:
     value = os.environ.get(ref.env_var, "")
     if value:

tools/chemenu/sites/__init__.py

@@ -16,7 +16,10 @@ host and path the stack has a handler for replaces it; the `sites` section of th
 settings still switches handlers off, sets the `locale` and adds hint lines
 (`notes`). A handler may also offer the subpages of a page
 (`Subpages`: the links to take, under a path prefix), which `raw fetch --scope`
-fetches as one bundle. A handler may still be registered from a Python module in this
+fetches as one bundle; name the login page of a website behind a login and how
+a page shows it arrived signed in (`Session`, for `raw login` and a fetch with
+the user's session); and declare that its pages are drawn by JavaScript
+(`render`). A handler may still be registered from a Python module in this
 package. The core knows neither `typer` nor `rich` and raises `ValidationError`.
 """
 from __future__ import annotations
@@ -123,12 +126,44 @@ class Subpages:
         return found
 
 
+@dataclass(frozen=True)
+class Session:
+    """What a website behind a login needs from the stack: the page `raw login`
+    opens for the user to sign in on, and how a fetched page shows whether it
+    arrived signed in. `signed_in` matches only on a signed-in page (a logout
+    link), `signed_out` only on a page that was not (a paywall teaser); with
+    both, both must hold. Whether a host's pages are fetched with the session at
+    all is the instance's choice (`fetch.routes`), not the handler's."""
+
+    login: str
+    signed_in: Optional[Rule] = None
+    signed_out: Optional[Rule] = None
+
+    def __post_init__(self) -> None:
+        if self.signed_in is None and self.signed_out is None:
+            raise ValueError("a session needs a signed_in or a signed_out rule - nothing would tell a teaser from the article")
+        if urllib.parse.urlsplit(self.login).scheme.lower() not in web_capture.ALLOWED_SCHEMES:
+            raise ValueError(f"a session's login page is an http(s) URL, not {self.login!r}")
+
+    def is_signed_in(self, html: str) -> bool:
+        document = web_capture.parse(html)
+        if self.signed_in is not None and not web_capture.count_matches(document, self.signed_in):
+            return False
+        if self.signed_out is not None and web_capture.count_matches(document, self.signed_out):
+            return False
+        return True
+
+
 @dataclass(frozen=True)
 class Site:
     """One handler. `host` is matched exactly (lower case); `path` is a regular
     expression matched against the start of the URL path, and the longest `path`
     wins among the handlers of a host. `parent` names a handler this one
-    inherits from: its rewrites, rules and checks come first."""
+    inherits from: its rewrites, rules and checks come first.
+
+    `session` is what a fetch with the user's own login needs; `render` says the
+    page draws its content with JavaScript, so it is fetched through a browser
+    and the header says `rendered: true` - never without this declaration."""
 
     name: str
     host: str
@@ -139,6 +174,8 @@ class Site:
     drop: tuple[Rule, ...] = ()
     checks: tuple[Check, ...] = ()
     subpages: Optional[Subpages] = None
+    session: Optional[Session] = None
+    render: bool = False
     origin: str = "stack"
 
     def __post_init__(self) -> None:
@@ -177,6 +214,8 @@ def inherit(candidate: Site, parent: Site) -> Site:
         drop=parent.drop + candidate.drop,
         checks=parent.checks + candidate.checks,
         subpages=candidate.subpages or parent.subpages,
+        session=candidate.session or parent.session,
+        render=candidate.render or parent.render,
     )
     if len(flat.checks) > MAX_CHECKS:
         raise ValueError(
@@ -239,6 +278,14 @@ def site_for(url: str, disabled: frozenset[str] = frozenset(), instance: tuple[S
     return max(found, key=lambda s: len(s.path), default=None)
 
 
+def login_site(host: str, disabled: frozenset[str] = frozenset(), instance: tuple[Site, ...] = ()) -> Optional[Site]:
+    """The enabled handler of `host` whose session `raw login` opens: among
+    those that declare one, the one with the shortest path pattern - the
+    website's own, not a section's."""
+    found = [s for s in catalogue(instance) if s.name not in disabled and s.host == host and s.session]
+    return min(found, key=lambda s: len(s.path), default=None)
+
+
 # --- the instance's part ----------------------------------------------------
 
 

tools/chemenu/sites/data.py

@@ -15,7 +15,7 @@ from typing import Any, Optional
 
 from jsonschema import Draft202012Validator
 
-from chemenu.sites import Broken, Check, Rewrite, Rule, Site, Subpages, inherit
+from chemenu.sites import Broken, Check, Rewrite, Rule, Session, Site, Subpages, inherit
 
 SCHEMA_PATH = Path(__file__).with_name("site.schema.json")
 STACK_DIRECTORY = Path(__file__).with_name("handlers")
@@ -77,6 +77,17 @@ def build(name: str, document: Any, origin: str) -> Site:
             subpages = Subpages(_rule(entry["links"]), entry.get("prefix", "{page}"))
         except ValueError as exc:
             raise ValueError(f"subpages: {exc}") from None
+    session = None
+    if "session" in document:
+        entry = document["session"]
+        try:
+            session = Session(
+                entry["login"],
+                _rule(entry["signed_in"]) if "signed_in" in entry else None,
+                _rule(entry["signed_out"]) if "signed_out" in entry else None,
+            )
+        except ValueError as exc:
+            raise ValueError(f"session: {exc}") from None
     try:
         return Site(
             name=name,
@@ -88,6 +99,8 @@ def build(name: str, document: Any, origin: str) -> Site:
             drop=tuple(_rule(entry) for entry in document.get("drop", [])),
             checks=tuple(checks),
             subpages=subpages,
+            session=session,
+            render=document.get("render", False),
             origin=origin,
         )
     except (ValueError, re.error) as exc:

tools/chemenu/sites/site.schema.json

@@ -46,7 +46,22 @@
         "links": {"$ref": "#/$defs/rule"},
         "prefix": {"type": "string", "pattern": "^(\\{page\\}|/)", "maxLength": 200}
       }
-    }
+    },
+    "session": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["login"],
+      "properties": {
+        "login": {"type": "string", "pattern": "^https?://", "maxLength": 400},
+        "signed_in": {"$ref": "#/$defs/rule"},
+        "signed_out": {"$ref": "#/$defs/rule"}
+      },
+      "anyOf": [
+        {"required": ["signed_in"]},
+        {"required": ["signed_out"]}
+      ]
+    },
+    "render": {"type": "boolean"}
   },
   "$defs": {
     "rule": {

tools/chemenu/tests/test_cli.py

@@ -341,11 +341,13 @@ def test_network_yes_is_exactly_the_commands_that_can_reach_outside_this_checkou
     since `--latest`, which asks the release feed and downloads from it, and `raw fetch`
     since it exists (Gitea #120) - its `--html` form stays offline, which does not turn the
     command back to `no`. `raw capture` and `raw status` reach a git remote (Gitea #177), and so
-    does `export guidelines --push` (Gitea #179). Pinned as an explicit
+    does `export guidelines --push` (Gitea #179). `raw login` opens a website's login page in a
+    browser (Gitea #169). Pinned as an explicit
     set so a command gaining or losing that reach is a deliberate edit here, not a silent
     drift between the property and what the command actually does."""
     expected = {
         "raw fetch",
+        "raw login",
         "raw capture",
         "raw status",
         "export guidelines",

tools/chemenu/tests/test_config_shapes.py

@@ -78,7 +78,7 @@ def test_the_telemetry_policy_reads_the_record():
 
 def test_one_record_per_section():
     assert [shape.key for shape in config_shapes.SETTINGS_SECTIONS] == [
-        "telemetry", "remotes", "upload", "tasks", "superproductivity", "caldav", "sites",
+        "telemetry", "remotes", "upload", "tasks", "superproductivity", "caldav", "sites", "fetch",
     ]
 
 

tools/chemenu/tests/test_doctor.py

@@ -740,3 +740,35 @@ def test_marked_scripts_fail_with_the_unblock_command(instance, monkeypatch):
 def test_unmarked_scripts_are_ok(instance, monkeypatch):
     _windows(monkeypatch)
     assert _status(doctor.run_doctor(), "script-marks") == "OK"
+
+
+# --- the optional browser component --------------------------------------------
+
+
+def _without_browser(monkeypatch) -> None:
+    from chemenu import browser
+
+    monkeypatch.setattr(browser, "missing", lambda: "the Python package `playwright` is not installed in the venv")
+
+
+def test_a_missing_browser_component_nothing_needs_is_ok(instance, monkeypatch):
+    _without_browser(monkeypatch)
+    checks = doctor.run_doctor()
+    assert _status(checks, "browser") == "OK"
+    assert "--with browser" in _detail(checks, "browser")
+
+
+def test_a_missing_browser_component_a_session_route_needs_is_a_warning_not_a_failure(instance, monkeypatch):
+    _without_browser(monkeypatch)
+    write_settings(config.ROOT, monkeypatch, fetch={"routes": {"www.example.org": "session"}})
+    checks = doctor.run_doctor()
+    assert _status(checks, "browser") == "WARN"
+    assert "www.example.org" in _detail(checks, "browser")
+    assert "--with browser" in _fix(checks, "browser")
+
+
+def test_an_installed_browser_component_is_ok(instance, monkeypatch):
+    from chemenu import browser
+
+    monkeypatch.setattr(browser, "missing", lambda: None)
+    assert _status(doctor.run_doctor(), "browser") == "OK"

tools/chemenu/tests/test_preflight.py

@@ -78,11 +78,24 @@ case "$1" in
         shift 2
         [ "$1" = --version ] && exit 0
         echo "$*" >> "$PIP_LOG"
-        if [ -n "${{FAKE_PIP_FAIL:-}}" ]; then
+        case "$*" in
+          *requirements-browser*) fail=${{FAKE_BROWSER_PIP_FAIL:-}} ;;
+          *) fail=${{FAKE_PIP_FAIL:-}} ;;
+        esac
+        if [ -n "$fail" ]; then
           echo "ERROR: Could not find a version that satisfies the requirement (network unreachable)" >&2
           exit 1
         fi
         exit 0 ;;
+      playwright)
+        shift 2
+        echo "playwright $* into $PLAYWRIGHT_BROWSERS_PATH" >> "$PIP_LOG"
+        if [ -n "${{FAKE_PLAYWRIGHT_FAIL:-}}" ]; then
+          echo "Host system is missing dependencies to run browsers" >&2
+          exit 1
+        fi
+        mkdir -p "$PLAYWRIGHT_BROWSERS_PATH/chromium-1200"
+        exit 0 ;;
     esac ;;
 esac
 exit 1
@@ -319,6 +332,69 @@ def test_pip_failure_stops_and_is_retried_next_time(machine, shell):
     assert machine.pip_log.read_text(encoding="utf-8").count("install") == 2
 
 
+# --- the optional browser component -------------------------------------------------
+
+
+def _browser_lines(machine) -> list[str]:
+    if not machine.pip_log.exists():
+        return []
+    return [line for line in machine.pip_log.read_text(encoding="utf-8").splitlines()
+            if "requirements-browser" in line or line.startswith("playwright ")]
+
+
+@pytest.mark.parametrize("shell", SHELLS)
+def test_without_with_browser_nothing_of_the_component_is_installed(machine, shell):
+    machine.standard()
+    (machine.tools / "requirements-browser.txt").write_text("playwright\n", encoding="utf-8")
+    result = machine.run(shell=shell)
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert _browser_lines(machine) == []
+    assert not any(line.split()[1:2] == ["browser"] for line in result.stdout.splitlines())
+
+
+@pytest.mark.parametrize("shell", SHELLS)
+def test_with_browser_installs_the_component_into_the_venv_and_keeps_it(machine, shell):
+    machine.standard()
+    (machine.tools / "requirements-browser.txt").write_text("playwright\n", encoding="utf-8")
+    result = machine.run("--with", "browser", shell=shell)
+    assert result.returncode == 0, result.stdout + result.stderr
+    venv = machine.tools / ".venv"
+    lines = _browser_lines(machine)
+    assert any("-r" in line and line.endswith("requirements-browser.txt") for line in lines)
+    assert f"playwright install chromium into {venv / 'ms-playwright'}" in lines
+    assert any(line.split()[:2] == ["OK", "browser"] for line in result.stdout.splitlines())
+    assert (venv / ".chemenu-browser.sha256").is_file()
+    # A later plain run keeps it, and reinstalls only when its requirements change.
+    assert machine.run(shell=shell).returncode == 0
+    assert len(_browser_lines(machine)) == 2
+    (machine.tools / "requirements-browser.txt").write_text("playwright>=2\n", encoding="utf-8")
+    result = machine.run(shell=shell)
+    assert result.returncode == 0 and len(_browser_lines(machine)) == 4
+    assert "OK       browser" in result.stdout
+
+
+@pytest.mark.parametrize("shell", SHELLS)
+@pytest.mark.parametrize("failing", ["FAKE_BROWSER_PIP_FAIL", "FAKE_PLAYWRIGHT_FAIL"])
+def test_a_component_that_fails_stops_but_leaves_wikitool_working(machine, shell, failing):
+    machine.standard()
+    (machine.tools / "requirements-browser.txt").write_text("playwright\n", encoding="utf-8")
+    machine.extra_env[failing] = "1"
+    result = machine.run("--with=browser", shell=shell)
+    assert result.returncode == 42
+    assert_guidance(result.stdout, "optional browser component", "--with browser")
+    assert machine.recorded()["complete"] is True
+    assert not (machine.tools / ".venv" / ".chemenu-browser.sha256").exists()
+
+
+@pytest.mark.parametrize("shell", SHELLS)
+def test_an_unknown_component_is_refused(machine, shell):
+    machine.standard()
+    result = machine.run("--with", "spreadsheet", shell=shell)
+    assert result.returncode == 1
+    assert "not an optional component" in result.stderr
+    assert not machine.tools_file.exists()
+
+
 # --- the store alias ----------------------------------------------------------------
 
 

tools/chemenu/tests/test_preflight_pwsh.py

@@ -183,6 +183,42 @@ def test_pip_failure_stops_and_is_retried_next_time(machine):
     assert machine.recorded()["complete"] is True
 
 
+def _browser_lines(machine: Machine) -> list[str]:
+    if not machine.pip_log.exists():
+        return []
+    return [line for line in machine.pip_log.read_text(encoding="utf-8").splitlines()
+            if "requirements-browser" in line or line.startswith("playwright ")]
+
+
+def test_with_browser_installs_the_component_and_a_plain_run_keeps_it(machine):
+    (machine.tools / "requirements-browser.txt").write_text("playwright\n", encoding="utf-8")
+    assert _preflight(machine).returncode == 0
+    assert _browser_lines(machine) == []
+    result = _preflight(machine, "--with", "browser")
+    assert result.returncode == 0, result.stdout + result.stderr
+    venv = machine.tools / ".venv"
+    assert f"playwright install chromium into {venv / 'ms-playwright'}" in _browser_lines(machine)
+    assert "OK       browser" in result.stdout
+    assert _preflight(machine).returncode == 0
+    assert len(_browser_lines(machine)) == 2
+
+
+@pytest.mark.parametrize("failing", ["FAKE_BROWSER_PIP_FAIL", "FAKE_PLAYWRIGHT_FAIL"])
+def test_a_component_that_fails_stops_but_leaves_wikitool_working(machine, failing):
+    (machine.tools / "requirements-browser.txt").write_text("playwright\n", encoding="utf-8")
+    machine.extra_env[failing] = "1"
+    result = _preflight(machine, "--with=browser")
+    assert result.returncode == 42
+    assert_guidance(result.stdout, "optional browser component", "--with browser")
+    assert machine.recorded()["complete"] is True
+
+
+def test_an_unknown_component_is_refused(machine):
+    result = _preflight(machine, "--with", "spreadsheet")
+    assert result.returncode == 1
+    assert "not an optional component" in result.stderr
+
+
 @pytest.mark.parametrize("platform, alias", [("linux", "python3"), ("windows", "python")])
 def test_store_alias_is_never_run_and_never_recorded(machine, platform, alias):
     marker = machine.base / "alias-was-run"

tools/chemenu/tests/test_raw_fetch_routes.py

@@ -0,0 +1,352 @@
+"""`wikitool raw fetch` along a route: with the user's session, through an
+external fetcher, or rendered by a browser (Gitea #169).
+
+As in `test_raw_fetch.py`, every request goes to a real `http.server` on
+127.0.0.1. The paid pages answer by the `Cookie` header they receive - the full
+article to the right session, a teaser to anything else - so what is under test
+is what a session actually sends, not what a stub was told. The browser is the
+one thing replaced: Playwright is an optional component that CI does not
+install, so `browser.render` and `browser.missing` are stood in for.
+"""
+from __future__ import annotations
+
+import datetime
+import json
+import os
+import sys
+from pathlib import Path
+
+import pytest
+import typer
+
+from chemenu import browser, config, config_shapes, fetch_routes, web_capture, web_session
+from chemenu.commands.raw.fetch import raw_fetch_command
+from chemenu.tests.conftest import write_settings
+from chemenu.tests.test_raw_fetch import ARTICLE, Site
+
+SECRET = "sid-6f3a-THIS-IS-THE-SESSION-VALUE"
+FULL = (
+    b"<!doctype html><html><head><meta charset=\"utf-8\"><title>Paid</title></head><body><main>"
+    b"<h1>The paid article</h1><p>All of the text, " + b"long enough to be an article. " * 10 + b"</p>"
+    b"<a class=\"logout\" href=\"/logout\">Log out</a></main></body></html>"
+)
+TEASER = (
+    b"<!doctype html><html><head><meta charset=\"utf-8\"><title>Paid</title></head><body><main>"
+    b"<h1>The paid article</h1><p>The first sentence.</p>"
+    b"<div class=\"paywall\">Subscribe to read on</div></main></body></html>"
+)
+HANDLER = {
+    "host": "127.0.0.1",
+    "summary": "A test publisher behind a login.",
+    "session": {
+        "login": "http://127.0.0.1/login",
+        "signed_in": {"tag": "a", "class": "logout"},
+        "signed_out": {"class": "paywall"},
+    },
+}
+
+
+def _state(value: str = SECRET, expires: float = -1) -> dict:
+    return {
+        "cookies": [{
+            "name": "sid", "value": value, "domain": "127.0.0.1", "path": "/", "expires": expires,
+            "httpOnly": True, "secure": False, "sameSite": "Lax",
+        }],
+        "origins": [],
+    }
+
+
+def _paid(server: Site, path: str = "/paid/article") -> str:
+    """A page that answers by the cookie it receives."""
+
+    def answer(handler) -> None:
+        handler.wfile.write(FULL if f"sid={SECRET}" in (handler.headers.get("Cookie") or "") else TEASER)
+
+    server.routes[path] = (200, {"Content-Type": "text/html; charset=utf-8"}, answer)
+    return server.url(path)
+
+
+@pytest.fixture
+def site(monkeypatch):
+    for var in ("http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"):
+        monkeypatch.delenv(var, raising=False)
+    server = Site()
+    yield server
+    server.stop()
+
+
+@pytest.fixture
+def tree(kb_dir, monkeypatch):
+    root = kb_dir.parent
+    (root / "raw").mkdir()
+    (root / "incoming").mkdir(exist_ok=True)
+    (root / "sites").mkdir()
+    return root
+
+
+def _handler(root: Path, **changes) -> None:
+    (root / "sites/publisher.json").write_text(json.dumps({**HANDLER, **changes}), encoding="utf-8")
+
+
+@pytest.fixture
+def routed(tree, monkeypatch):
+    """127.0.0.1 routed through the session, with a handler that names the login."""
+    _handler(tree)
+    write_settings(tree, monkeypatch, fetch={"routes": {"127.0.0.1": "session"}})
+    return tree
+
+
+def _session_env(monkeypatch, state: dict) -> None:
+    monkeypatch.setenv(config.ENV_SECRET_PREFIX + web_session.secret_name("127.0.0.1").upper(), json.dumps(state))
+
+
+def _fetch(url, name=None, scope=False):
+    return raw_fetch_command(url=url, html=None, source_url=None, name=name, scope=scope)
+
+
+def _exit_code(url, **kwargs) -> int:
+    with pytest.raises(typer.Exit) as excinfo:
+        _fetch(url, **kwargs)
+    return excinfo.value.exit_code
+
+
+def _header(md: Path) -> dict:
+    import yaml
+
+    content = md.read_text(encoding="utf-8")
+    return yaml.safe_load(content[4:].partition("\n---\n")[0])
+
+
+def _snapshot(root: Path) -> dict[str, bytes]:
+    return {str(p.relative_to(root)): p.read_bytes() for p in sorted(root.rglob("*")) if p.is_file()}
+
+
+# --- with the user's session ------------------------------------------------
+
+
+def test_with_a_stored_session_the_full_text_arrives_byte_identical(routed, site, monkeypatch, capsys):
+    _session_env(monkeypatch, _state())
+    url = _paid(site)
+    _fetch(url)
+    out = capsys.readouterr().out
+    assert (routed / "incoming/article.html").read_bytes() == FULL
+    header = _header(routed / "incoming/article.md")
+    assert header["session"] == "127.0.0.1"
+    assert header["fetched_by"] == "wikitool raw fetch"
+    assert "rendered" not in header
+    assert SECRET not in (routed / "incoming/article.md").read_text(encoding="utf-8")
+    assert SECRET not in out
+    assert "raw fetch --html" not in out  # the teaser hint is for the anonymous fetch
+
+
+def test_the_session_is_read_from_the_secrets_file_outside_the_tree(routed, site, monkeypatch):
+    path = web_session.store("127.0.0.1", "testinstance", _state())
+    assert path == config.secrets_file("testinstance")  # the user's configuration directory
+    if os.name != "nt":
+        assert path.stat().st_mode & 0o077 == 0
+    _fetch(_paid(site))
+    assert (routed / "incoming/article.html").read_bytes() == FULL
+    # The test's HOME lies inside its tmp_path; in it, the secrets file is the one place the value is.
+    holding = [p for p in routed.rglob("*") if p.is_file() and SECRET.encode() in p.read_bytes()]
+    assert holding == [path]
+
+
+@pytest.mark.parametrize("state", [None, _state("stale"), _state(expires=1_000_000_000)],
+                         ids=["no-session", "ended-by-the-website", "expired-cookie"])
+def test_without_a_valid_session_exit_42_names_raw_login_and_keeps_no_teaser(routed, site, monkeypatch, capsys, state):
+    if state is not None:
+        _session_env(monkeypatch, state)
+    url = _paid(site)
+    before = _snapshot(routed / "incoming")
+    assert _exit_code(url) == 42
+    out = capsys.readouterr().out
+    assert "tools/wikitool raw login 127.0.0.1" in out
+    assert _snapshot(routed / "incoming") == before
+    if state is None:
+        assert site.requests == []  # nothing to send, so nothing is asked
+
+
+def test_a_session_route_without_a_login_in_any_handler_is_refused_before_fetching(tree, site, monkeypatch, capsys):
+    (tree / "sites/publisher.json").write_text(
+        json.dumps({k: v for k, v in HANDLER.items() if k != "session"}), encoding="utf-8")
+    write_settings(tree, monkeypatch, fetch={"routes": {"127.0.0.1": "session"}})
+    assert _exit_code(_paid(site)) == 1
+    assert "names a login page" in capsys.readouterr().out
+    assert site.requests == []
+
+
+def test_scope_sends_the_session_with_every_page_and_names_it(routed, site, monkeypatch):
+    _handler(routed, path="/paid/", subpages={"links": {"tag": "a", "inside": "td"}})
+    write_settings(routed, monkeypatch, sites={"pause_seconds": 0})
+    _session_env(monkeypatch, _state())
+    index = (b"<html><body><main><h1>Index</h1><table><tr><td><a href=\"one\">One</a></td></tr></table>"
+             b"<a class=\"logout\" href=\"/logout\">Log out</a></main></body></html>")
+
+    def answer(handler) -> None:
+        handler.wfile.write(index if f"sid={SECRET}" in (handler.headers.get("Cookie") or "") else TEASER)
+
+    site.routes["/paid/"] = (200, {"Content-Type": "text/html"}, answer)
+    _paid(site, "/paid/one")
+    _fetch(site.url("/paid/"), name="bundle", scope=True)
+    bundle = routed / "incoming/bundle"
+    assert (bundle / "one.html").read_bytes() == FULL
+    assert _header(bundle / "one.md")["session"] == "127.0.0.1"
+    assert json.loads((bundle / "_scope.json").read_text(encoding="utf-8"))["session"] == "127.0.0.1"
+
+
+# --- rendered by a browser --------------------------------------------------
+
+
+def _fake_render(calls: list):
+    def render(url, user_agent, state=None, **kwargs):
+        calls.append((url, state))
+        return web_capture.Response(url, url, 200, "text/html; charset=utf-8", FULL)
+    return render
+
+
+def test_a_handler_declaring_render_gives_a_rendered_header(tree, site, monkeypatch):
+    _handler(tree, render=True)
+    calls: list = []
+    monkeypatch.setattr(browser, "render", _fake_render(calls))
+    url = site.html("/page", ARTICLE)
+    _fetch(url)
+    assert calls == [(url, None)]
+    assert site.requests == []  # the browser fetched it, not the built-in fetch
+    assert _header(tree / "incoming/page.md")["rendered"] is True
+
+
+def test_render_with_a_session_hands_the_browser_the_session(routed, site, monkeypatch):
+    _handler(routed, render=True)
+    _session_env(monkeypatch, _state())
+    calls: list = []
+    monkeypatch.setattr(browser, "render", _fake_render(calls))
+    _fetch(site.url("/paid/article"))
+    assert calls[0][1] == _state()
+    header = _header(routed / "incoming/article.md")
+    assert header["rendered"] is True and header["session"] == "127.0.0.1"
+
+
+def test_without_the_declaration_nothing_is_ever_rendered(routed, site, monkeypatch):
+    _session_env(monkeypatch, _state())
+
+    def refuse(*args, **kwargs):
+        raise AssertionError("rendered without a declaration")
+
+    monkeypatch.setattr(browser, "render", refuse)
+    _fetch(_paid(site))
+    assert "rendered" not in _header(routed / "incoming/article.md")
+
+
+def test_render_without_the_browser_component_exits_42_with_the_preflight_option(tree, site, monkeypatch, capsys):
+    _handler(tree, render=True)
+    monkeypatch.setattr(browser, "missing", lambda: "the Python package `playwright` is not installed in the venv")
+    before = _snapshot(tree / "incoming")
+    assert _exit_code(site.html("/page", ARTICLE)) == 42
+    assert "tools/preflight.sh --with browser" in capsys.readouterr().out
+    assert _snapshot(tree / "incoming") == before
+
+
+# --- an external fetcher ----------------------------------------------------
+
+
+def _fetcher(tmp_path: Path, body: bytes = FULL, code: int = 0, stderr: str = "") -> list[str]:
+    script = tmp_path / "fetcher.py"
+    script.write_text(
+        "import sys\n"
+        f"sys.stdout.buffer.write({body!r})\n"
+        f"sys.stderr.write({stderr!r})\n"
+        f"open({str(tmp_path / 'fetcher.args')!r}, 'w').write(' '.join(sys.argv[1:]))\n"
+        f"sys.exit({code})\n",
+        encoding="utf-8",
+    )
+    return [sys.executable, str(script), "--html"]
+
+
+def test_an_external_fetcher_writes_its_stdout_byte_identical(tree, site, monkeypatch, tmp_path):
+    write_settings(tree, monkeypatch, fetch={"routes": {"127.0.0.1": "homelab"},
+                                             "fetchers": {"homelab": _fetcher(tmp_path)}})
+    url = site.url("/elsewhere/post")
+    _fetch(url)
+    assert site.requests == []  # the built-in fetch never ran
+    assert (tmp_path / "fetcher.args").read_text(encoding="utf-8") == f"--html {url}"
+    assert (tree / "incoming/post.html").read_bytes() == FULL
+    header = _header(tree / "incoming/post.md")
+    assert header["fetched_by"] == "wikitool raw fetch (fetcher: homelab)"
+    assert "http_status" not in header
+
+
+def test_a_failing_fetcher_exits_1_with_its_stderr_and_no_fallback(tree, site, monkeypatch, tmp_path, capsys):
+    write_settings(tree, monkeypatch, fetch={
+        "routes": {"127.0.0.1": "homelab"},
+        "fetchers": {"homelab": _fetcher(tmp_path, code=3, stderr="upstream says 503")},
+    })
+    url = site.html("/post", ARTICLE)
+    before = _snapshot(tree / "incoming")
+    assert _exit_code(url) == 1
+    out = capsys.readouterr().out
+    assert "exit 3" in out and "upstream says 503" in out
+    assert _snapshot(tree / "incoming") == before
+    assert site.requests == []
+
+
+def test_a_route_to_a_fetcher_this_checkout_lacks_is_refused(tree, site, monkeypatch, capsys):
+    write_settings(tree, monkeypatch, fetch={"routes": {"127.0.0.1": "homelab"}})
+    assert _exit_code(site.html("/post", ARTICLE)) == 1
+    assert "fetch.fetchers.homelab" in capsys.readouterr().out
+    assert site.requests == []
+
+
+def test_a_fetcher_command_in_the_committed_file_is_refused(tree, site, monkeypatch, capsys, tmp_path):
+    (tree / config.INSTANCE_SETTINGS_FILENAME).write_text(json.dumps({
+        "schema": 1, "instance_id": "testinstance",
+        "fetch": {"routes": {"127.0.0.1": "homelab"}, "fetchers": {"homelab": _fetcher(tmp_path)}},
+    }), encoding="utf-8")
+    assert _exit_code(site.html("/post", ARTICLE)) == 1
+    out = capsys.readouterr().out
+    assert "fetch.fetchers" in out and "checkout" in out
+    assert site.requests == []
+    assert config_shapes.FETCH.field("fetchers").names_command
+    with pytest.raises(ValueError):
+        config_shapes.Field("fetchers", "object", "", tier=config_shapes.INSTANCE, names_command=True)
+
+
+# --- a host no route names --------------------------------------------------
+
+
+def _without_time(md: Path) -> str:
+    return "\n".join(line for line in md.read_text(encoding="utf-8").splitlines() if not line.startswith("retrieved:"))
+
+
+def test_a_host_no_route_names_is_fetched_exactly_as_before(tree, site, monkeypatch, capsys):
+    url = site.html("/post", ARTICLE)
+    _fetch(url, name="first")
+    first_out = capsys.readouterr().out
+    write_settings(tree, monkeypatch, fetch={"routes": {"other.example.org": "session"}})
+    _fetch(url, name="later")
+    later_out = capsys.readouterr().out
+    assert (tree / "incoming/later.html").read_bytes() == (tree / "incoming/first.html").read_bytes()
+    assert _without_time(tree / "incoming/later.md") == _without_time(tree / "incoming/first.md").replace(
+        "first.html", "later.html")
+    assert later_out.replace("later", "first") == first_out
+    assert [ua for _, ua in site.requests] == [site.requests[0][1]] * 2
+
+
+# --- the pieces -------------------------------------------------------------
+
+
+def test_secret_names_are_valid_reference_names():
+    assert web_session.secret_name("www.heise.de") == "session_www_heise_de"
+    assert web_session.secret_name("a--b.example") == "session_a_b_example"
+
+
+def test_expired_cookies_are_not_sent(site):
+    jar = web_session.cookie_jar(_state(expires=(datetime.datetime.now() - datetime.timedelta(days=1)).timestamp()))
+    url = _paid(site)
+    response = web_capture.fetch(url, "test", cookies=jar)
+    assert response.body == TEASER
+
+
+def test_a_route_names_a_session_label_only_for_the_session():
+    assert fetch_routes.Route("h", fetch_routes.SESSION).session_label == "h"
+    assert fetch_routes.Route("h", fetch_routes.FETCHER, fetcher="x").session_label == ""
+    assert fetch_routes.Route("h", fetch_routes.FETCHER, fetcher="x").fetched_by.endswith("(fetcher: x)")

tools/chemenu/tests/test_raw_login.py

@@ -0,0 +1,124 @@
+"""`wikitool raw login <host>` - the user signs in once, and the session is stored
+as a secret outside the working tree (Gitea #169).
+
+The browser window itself needs a human and the optional Playwright component,
+so `browser.login` and `browser.missing` are stood in for: what is under test is
+the order of the checks, where the session lands, and that nothing prints it.
+"""
+from __future__ import annotations
+
+import json
+import os
+
+import pytest
+import typer
+
+from chemenu import browser, config, web_session
+from chemenu.commands.raw.fetch import raw_fetch_command
+from chemenu.commands.raw.login import raw_login_command
+from chemenu.tests.conftest import write_settings
+from chemenu.tests.test_raw_fetch_routes import FULL, HANDLER, SECRET, _paid, _state, site, tree  # noqa: F401
+
+
+@pytest.fixture
+def routed(tree, monkeypatch):  # noqa: F811
+    (tree / "sites/publisher.json").write_text(json.dumps(HANDLER), encoding="utf-8")
+    write_settings(tree, monkeypatch, fetch={"routes": {"127.0.0.1": "session"}})
+    return tree
+
+
+def _login(host: str = "127.0.0.1") -> int:
+    try:
+        raw_login_command(host=host)
+    except typer.Exit as exc:
+        return exc.exit_code
+    return 0
+
+
+def test_login_stores_the_session_outside_the_tree_and_prints_none_of_it(routed, site, monkeypatch, capsys):  # noqa: F811
+    opened = []
+    monkeypatch.setattr(browser, "login", lambda url: opened.append(url) or _state())
+    assert _login() == 0
+    out = capsys.readouterr()
+    assert opened == ["http://127.0.0.1/login"]
+    path = config.secrets_file("testinstance")
+    stored = json.loads(json.loads(path.read_text(encoding="utf-8"))["session_127_0_0_1"])
+    assert stored == _state()
+    if os.name != "nt":
+        assert path.stat().st_mode & 0o077 == 0
+    assert SECRET not in out.out + out.err
+    assert "session_127_0_0_1" in out.out
+    # ... and the next fetch uses it.
+    raw_fetch_command(url=_paid(site), html=None, source_url=None, name=None, scope=False)
+    assert (routed / "incoming/article.html").read_bytes() == FULL
+
+
+def test_login_keeps_the_other_secrets(routed, monkeypatch):
+    path = config.secrets_file("testinstance")
+    path.parent.mkdir(parents=True)
+    path.write_text(json.dumps({"caldav_app_password": "other"}), encoding="utf-8")
+    path.chmod(0o600)
+    monkeypatch.setattr(browser, "login", lambda url: _state())
+    assert _login() == 0
+    assert json.loads(path.read_text(encoding="utf-8"))["caldav_app_password"] == "other"
+
+
+def test_login_takes_a_url_for_its_host(routed, monkeypatch):
+    monkeypatch.setattr(browser, "login", lambda url: _state())
+    assert _login("http://127.0.0.1/some/article") == 0
+    assert web_session.load("127.0.0.1", "testinstance") == _state()
+
+
+def test_login_without_the_browser_component_exits_42_with_the_preflight_option(routed, monkeypatch, capsys):
+    monkeypatch.setattr(browser, "missing", lambda: "the Python package `playwright` is not installed in the venv")
+    assert _login() == 42
+    out = capsys.readouterr().out
+    assert "tools/preflight.sh --with browser" in out
+    assert not config.secrets_file("testinstance").exists()
+
+
+def test_login_for_a_host_not_routed_through_the_session_is_refused(tree, monkeypatch, capsys):  # noqa: F811
+    (tree / "sites/publisher.json").write_text(json.dumps(HANDLER), encoding="utf-8")
+    monkeypatch.setattr(browser, "login", lambda url: pytest.fail("the browser was opened"))
+    assert _login() == 1
+    assert "config set fetch.routes" in capsys.readouterr().out
+
+
+def test_login_for_a_host_whose_handler_names_no_login_page_is_refused(tree, monkeypatch, capsys):  # noqa: F811
+    write_settings(tree, monkeypatch, fetch={"routes": {"127.0.0.1": "session"}})
+    monkeypatch.setattr(browser, "login", lambda url: pytest.fail("the browser was opened"))
+    assert _login() == 1
+    assert "names a login page" in capsys.readouterr().out
+
+
+def test_login_warns_when_a_variable_answers_before_the_file(routed, monkeypatch, capsys):
+    monkeypatch.setenv("WIKITOOL_SECRET_SESSION_127_0_0_1", json.dumps(_state("older")))
+    monkeypatch.setattr(browser, "login", lambda url: _state())
+    assert _login() == 0
+    assert "WARN WIKITOOL_SECRET_SESSION_127_0_0_1 is set" in capsys.readouterr().out
+
+
+def test_the_component_is_reported_missing_without_playwright(monkeypatch):
+    monkeypatch.setattr(browser.importlib.util, "find_spec", lambda name: None)
+    assert "playwright" in browser.missing()
+    with pytest.raises(browser.BrowserMissing, match="--with browser"):
+        browser.require("`raw login`")
+
+
+def test_config_set_writes_a_route_and_leaves_a_fetcher_to_the_user(tree):  # noqa: F811
+    from typer.testing import CliRunner
+
+    from chemenu.cli import app
+
+    runner = CliRunner()
+    result = runner.invoke(app, ["config", "set", "fetch.routes", '{"www.example.org": "session"}'])
+    assert result.exit_code == 0, result.output
+    data = json.loads((tree / config.INSTANCE_SETTINGS_FILENAME).read_text(encoding="utf-8"))
+    assert data["fetch"] == {"routes": {"www.example.org": "session"}} and data["instance_id"]
+
+    result = runner.invoke(app, ["config", "set", "fetch.fetchers", '{"homelab": ["/bin/true"]}'])
+    assert result.exit_code == 42
+    assert not (tree / config.LOCAL_SETTINGS_FILENAME).exists()
+
+    result = runner.invoke(app, ["config", "set", "fetch.routes", '{"www.example.org": "Session!"}'])
+    assert result.exit_code == 1

tools/chemenu/tests/test_site_files.py

@@ -190,6 +190,10 @@ BROKEN = {
     "a placeholder without its field": {**EXAMPLE, "checks": [{"text": "see {count}"}]},
     "a parent that does not exist": {**EXAMPLE, "parent": "nobody"},
     "a parent for another host": {**EXAMPLE, "parent": "learn-microsoft"},
+    "a session without a sign of being signed in": {**EXAMPLE, "session": {"login": "https://docs.example.org/login"}},
+    "a session whose login page is not http(s)": {
+        **EXAMPLE, "session": {"login": "file:///etc/passwd", "signed_in": {"class": "logout"}}},
+    "a render that is not a boolean": {**EXAMPLE, "render": "yes"},
 }
 
 
@@ -273,6 +277,20 @@ def test_a_symlink_is_not_followed(tree):
     assert doctor.check_site_handlers().status == "FAIL"
 
 
+def test_a_session_and_render_are_read_and_inherited(tree):
+    _handler(tree, "example", {**EXAMPLE, "render": True, "session": {
+        "login": "https://docs.example.org/login", "signed_out": {"class": "paywall"}}})
+    _handler(tree, "example-guide", {"host": "docs.example.org", "summary": "The guide.", "path": "/guide/",
+                                     "parent": "example"})
+    instance, broken = sites.read_instance(tree)
+    assert broken == ()
+    child = next(s for s in instance if s.name == "example-guide")
+    assert child.render and child.session.login == "https://docs.example.org/login"
+    assert child.session.is_signed_in("<p>article</p>")
+    assert not child.session.is_signed_in('<div class="paywall">subscribe</div>')
+    assert sites.login_site("docs.example.org", instance=instance).name == "example"
+
+
 def test_doctor_is_quiet_without_a_sites_directory(tree):
     assert doctor.check_site_handlers().status == "OK"
 

tools/chemenu/web_capture.py

@@ -20,6 +20,7 @@ from __future__ import annotations
 import codecs
 import datetime
 import email.message
+import http.cookiejar
 import json
 import mimetypes
 import re
@@ -109,11 +110,14 @@ def fetch(
     user_agent: str,
     timeout: float = TIMEOUT_SECONDS,
     max_bytes: int = MAX_BYTES,
+    cookies: Optional[http.cookiejar.CookieJar] = None,
 ) -> Response:
     """GET `url` and return the body exactly as received.
 
-    No cookies (the opener carries no cookie processor) and no content
-    decoding - urllib asks for `identity`, so the bytes are the page, not a
+    No cookies unless `cookies` is given - the user's session for a website
+    behind a login (`chemenu.web_session`), sent by the jar's own domain, path,
+    `secure` and expiry rules, on every redirect hop too - and no content
+    decoding: urllib asks for `identity`, so the bytes are the page, not a
     compressed transfer of it. `timeout` bounds the whole transfer, not only a
     single read, so a server that trickles bytes cannot hold the call open.
     Raises `ValidationError` for a URL outside `http`/`https` (also on a
@@ -121,7 +125,10 @@ def fetch(
     error status, an unreachable host, a timeout, a body over `max_bytes`.
     """
     check_url(url)
-    opener = urllib.request.build_opener(_HttpOnlyRedirects)
+    handlers: list = [_HttpOnlyRedirects]
+    if cookies is not None:
+        handlers.append(urllib.request.HTTPCookieProcessor(cookies))
+    opener = urllib.request.build_opener(*handlers)
     request = urllib.request.Request(url, headers={"User-Agent": user_agent, "Accept": "*/*"})
     deadline = time.monotonic() + timeout
     try:
@@ -712,6 +719,9 @@ def derive_document(
     drop: "tuple[Rule, ...]" = (),
     site: str = "",
     requested_url: str = "",
+    fetched_by: str = "wikitool raw fetch",
+    session: str = "",
+    rendered: bool = False,
 ) -> Derivation:
     """The `.md` beside a captured `.html`: the fixed header, then the derived
     text. With `response` it describes a fetch; without, a page a human saved
@@ -719,22 +729,29 @@ def derive_document(
 
     `drop`, `site` and `requested_url` come from a site handler
     (`chemenu.sites`): its rules for the text, its name, and the URL the user
-    typed when the page was fetched under another one."""
+    typed when the page was fetched under another one. `fetched_by`, `session`
+    and `rendered` come from the route the fetch took (`chemenu.fetch_routes`):
+    an external fetcher names itself, a fetch with the user's session names the
+    host whose session it was - never its content - and a page drawn by a
+    browser says so. A response without an HTTP status (an external fetcher's)
+    leaves `http_status` out rather than inventing one."""
     decoded = decode(body, response.content_type if response else None)
     document_title = title_of(parse(decoded.text))
     text = derive_text(decoded.text, base_url, drop)
     charset = f"{decoded.charset} (from: {decoded.source})"
     if response is not None:
         fields = [
-            ("fetched_by", "wikitool raw fetch"),
+            ("fetched_by", fetched_by),
             ("url", response.url),
             *([("requested_url", requested_url)] if requested_url else []),
             ("final_url", response.final_url),
             ("retrieved", derived_at or utc_now()),
-            ("http_status", str(response.status)),
+            *([("http_status", str(response.status))] if response.status else []),
             ("content_type", response.content_type),
             ("charset", charset),
             *([("site", site)] if site else []),
+            *([("session", session)] if session else []),
+            *([("rendered", "true")] if rendered else []),
             ("title", document_title),
             ("derived_from", html_name),
         ]

tools/chemenu/web_session.py

@@ -0,0 +1,115 @@
+"""The user's own session for a website behind a login, kept as a secret.
+
+`raw login <host>` lets the user sign in in a browser window and hands the
+resulting browser state - cookies and storage, in Playwright's `storage_state`
+form - to `store`. It lands in the secrets file of the instance
+(`config.secrets_file`), outside every working tree, under a name derived from
+the host (`secret_name`). `raw fetch` reads it back with `load` and sends its
+cookies through `cookie_jar`, with the built-in fetch and no browser.
+
+The session is never written to 1Password: it changes on every sign-in, and the
+stack only ever reads from there. `WIKITOOL_SECRET_<NAME>` stands in for the
+file as it does for any named secret, which is how a container gets a session
+without a secrets file.
+
+Nothing here prints a value. An error names the host, the secret's name and the
+file, never what is in it.
+"""
+from __future__ import annotations
+
+import http.cookiejar
+import json
+import os
+import re
+from pathlib import Path
+from typing import Any, Optional
+
+from chemenu import config, settings
+from chemenu.errors import ValidationError
+
+PREFIX = "session"
+_NON_ALNUM = re.compile(r"[^a-z0-9]+")
+
+
+def secret_name(host: str) -> str:
+    """The name the session of `host` is stored under: `session_` and the host
+    with every run of other characters as one `_` - `www.heise.de` becomes
+    `session_www_heise_de`, readable from `WIKITOOL_SECRET_SESSION_WWW_HEISE_DE`."""
+    parts = [part for part in _NON_ALNUM.split(host.lower()) if part]
+    if not parts:
+        raise ValidationError(f"{host!r} is not a host name")
+    return "_".join([PREFIX, *parts])
+
+
+def _env_var(host: str) -> str:
+    return config.ENV_SECRET_PREFIX + secret_name(host).upper()
+
+
+def _parse(raw: str, where: str) -> dict:
+    try:
+        state = json.loads(raw)
+    except ValueError:
+        raise ValidationError(f"the session in {where} is not a browser state (not JSON) - sign in again") from None
+    if not isinstance(state, dict) or not isinstance(state.get("cookies", []), list):
+        raise ValidationError(f"the session in {where} is not a browser state - sign in again")
+    return state
+
+
+def load(host: str, instance_id: Optional[str]) -> Optional[dict]:
+    """The stored browser state for `host`, or None when there is none.
+    Raises `ValidationError` for one that cannot be read - a secrets file
+    others can read, or a value that is not a browser state."""
+    variable = _env_var(host)
+    raw = os.environ.get(variable, "")
+    if raw:
+        return _parse(raw, variable)
+    if not instance_id:
+        return None
+    path = config.secrets_file(instance_id)
+    raw = (settings.read_secrets_file(path) or {}).get(secret_name(host), "")
+    if not raw:
+        return None
+    return _parse(raw, f"{path} ({secret_name(host)})")
+
+
+def store(host: str, instance_id: str, state: dict) -> Path:
+    """Write the browser state for `host` into the secrets file; returns its path."""
+    return settings.write_secret(instance_id, secret_name(host), json.dumps(state, separators=(",", ":")))
+
+
+def overridden(host: str) -> Optional[str]:
+    """The variable that answers for `host` before the secrets file, when it is set."""
+    variable = _env_var(host)
+    return variable if os.environ.get(variable) else None
+
+
+def _cookie(entry: Any) -> Optional[http.cookiejar.Cookie]:
+    if not isinstance(entry, dict):
+        return None
+    name, value, domain = entry.get("name"), entry.get("value"), entry.get("domain")
+    if not (isinstance(name, str) and isinstance(value, str) and isinstance(domain, str) and name and domain):
+        return None
+    expires = entry.get("expires", -1)
+    lasting = isinstance(expires, (int, float)) and not isinstance(expires, bool) and expires >= 0
+    dotted = domain.startswith(".")
+    return http.cookiejar.Cookie(
+        version=0, name=name, value=value, port=None, port_specified=False,
+        domain=domain, domain_specified=dotted, domain_initial_dot=dotted,
+        path=str(entry.get("path") or "/"), path_specified=True,
+        secure=bool(entry.get("secure", False)),
+        expires=int(expires) if lasting else None, discard=not lasting,
+        comment=None, comment_url=None,
+        rest={"HttpOnly": ""} if entry.get("httpOnly") else {},
+    )
+
+
+def cookie_jar(state: dict) -> http.cookiejar.CookieJar:
+    """A jar holding the cookies of `state`. Which of them a request carries
+    is the jar's decision - domain, path, `secure` and expiry - so an expired
+    cookie is simply not sent, and the page shows whether that mattered."""
+    jar = http.cookiejar.CookieJar()
+    for entry in state.get("cookies", []):
+        cookie = _cookie(entry)
+        if cookie is not None:
+            jar.set_cookie(cookie)
+    return jar