raw: fetch behind a login - raw login stores the user's session as a secret outside the tree, fetch.routes sends a host through it or to an external fetcher, a handler's render goes through the optional browser component (preflight --with browser) (#169); 10.0.0-beta.8

Files changed:
- CHANGES.md
- INSTALL.md
- VERSION
- docs/how-to/README.md
- docs/how-to/fetch-pages-behind-a-login.md
- docs/how-to/ingest-a-source.md
- docs/reference/commands/README.md
- docs/reference/commands/instance-health.md
- docs/reference/commands/raw-material-and-uploads.md
- docs/reference/configuration.md
- docs/reference/exit-codes.md
- instructions/preflight.md
- instructions/wiki-ingest/SKILL.md
- raw/CONTRACT.md
- sites/CONTRACT.md
- tools/CONTRACT.md
- tools/README.md
- tools/chemenu/browser.py
- tools/chemenu/cli_contract.py
- tools/chemenu/commands/doctor.py
- tools/chemenu/commands/raw/__init__.py
- tools/chemenu/commands/raw/fetch.py
- tools/chemenu/commands/raw/login.py
- tools/chemenu/config_shapes.py
- tools/chemenu/fetch_routes.py
- tools/chemenu/reference_doc.py
- tools/chemenu/settings.py
- tools/chemenu/sites/__init__.py
- tools/chemenu/sites/data.py
- tools/chemenu/sites/site.schema.json
- tools/chemenu/tests/test_cli.py
- tools/chemenu/tests/test_config_shapes.py
- tools/chemenu/tests/test_doctor.py
- tools/chemenu/tests/test_preflight.py
- tools/chemenu/tests/test_preflight_pwsh.py
- tools/chemenu/tests/test_raw_fetch_routes.py
- tools/chemenu/tests/test_raw_login.py
- tools/chemenu/tests/test_site_files.py
- tools/chemenu/web_capture.py
- tools/chemenu/web_session.py
- tools/preflight.ps1
- tools/preflight.sh
- tools/requirements-browser.txt

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2

tools/chemenu/tests/test_cli.py

@@ -341,11 +341,13 @@ def test_network_yes_is_exactly_the_commands_that_can_reach_outside_this_checkou
     since `--latest`, which asks the release feed and downloads from it, and `raw fetch`
     since it exists (Gitea #120) - its `--html` form stays offline, which does not turn the
     command back to `no`. `raw capture` and `raw status` reach a git remote (Gitea #177), and so
-    does `export guidelines --push` (Gitea #179). Pinned as an explicit
+    does `export guidelines --push` (Gitea #179). `raw login` opens a website's login page in a
+    browser (Gitea #169). Pinned as an explicit
     set so a command gaining or losing that reach is a deliberate edit here, not a silent
     drift between the property and what the command actually does."""
     expected = {
         "raw fetch",
+        "raw login",
         "raw capture",
         "raw status",
         "export guidelines",

tools/chemenu/tests/test_config_shapes.py

@@ -78,7 +78,7 @@ def test_the_telemetry_policy_reads_the_record():
 
 def test_one_record_per_section():
     assert [shape.key for shape in config_shapes.SETTINGS_SECTIONS] == [
-        "telemetry", "remotes", "upload", "tasks", "superproductivity", "caldav", "sites",
+        "telemetry", "remotes", "upload", "tasks", "superproductivity", "caldav", "sites", "fetch",
     ]
 
 

tools/chemenu/tests/test_doctor.py

@@ -740,3 +740,35 @@ def test_marked_scripts_fail_with_the_unblock_command(instance, monkeypatch):
 def test_unmarked_scripts_are_ok(instance, monkeypatch):
     _windows(monkeypatch)
     assert _status(doctor.run_doctor(), "script-marks") == "OK"
+
+
+# --- the optional browser component --------------------------------------------
+
+
+def _without_browser(monkeypatch) -> None:
+    from chemenu import browser
+
+    monkeypatch.setattr(browser, "missing", lambda: "the Python package `playwright` is not installed in the venv")
+
+
+def test_a_missing_browser_component_nothing_needs_is_ok(instance, monkeypatch):
+    _without_browser(monkeypatch)
+    checks = doctor.run_doctor()
+    assert _status(checks, "browser") == "OK"
+    assert "--with browser" in _detail(checks, "browser")
+
+
+def test_a_missing_browser_component_a_session_route_needs_is_a_warning_not_a_failure(instance, monkeypatch):
+    _without_browser(monkeypatch)
+    write_settings(config.ROOT, monkeypatch, fetch={"routes": {"www.example.org": "session"}})
+    checks = doctor.run_doctor()
+    assert _status(checks, "browser") == "WARN"
+    assert "www.example.org" in _detail(checks, "browser")
+    assert "--with browser" in _fix(checks, "browser")
+
+
+def test_an_installed_browser_component_is_ok(instance, monkeypatch):
+    from chemenu import browser
+
+    monkeypatch.setattr(browser, "missing", lambda: None)
+    assert _status(doctor.run_doctor(), "browser") == "OK"

tools/chemenu/tests/test_preflight.py

@@ -78,11 +78,24 @@ case "$1" in
         shift 2
         [ "$1" = --version ] && exit 0
         echo "$*" >> "$PIP_LOG"
-        if [ -n "${{FAKE_PIP_FAIL:-}}" ]; then
+        case "$*" in
+          *requirements-browser*) fail=${{FAKE_BROWSER_PIP_FAIL:-}} ;;
+          *) fail=${{FAKE_PIP_FAIL:-}} ;;
+        esac
+        if [ -n "$fail" ]; then
           echo "ERROR: Could not find a version that satisfies the requirement (network unreachable)" >&2
           exit 1
         fi
         exit 0 ;;
+      playwright)
+        shift 2
+        echo "playwright $* into $PLAYWRIGHT_BROWSERS_PATH" >> "$PIP_LOG"
+        if [ -n "${{FAKE_PLAYWRIGHT_FAIL:-}}" ]; then
+          echo "Host system is missing dependencies to run browsers" >&2
+          exit 1
+        fi
+        mkdir -p "$PLAYWRIGHT_BROWSERS_PATH/chromium-1200"
+        exit 0 ;;
     esac ;;
 esac
 exit 1
@@ -319,6 +332,69 @@ def test_pip_failure_stops_and_is_retried_next_time(machine, shell):
     assert machine.pip_log.read_text(encoding="utf-8").count("install") == 2
 
 
+# --- the optional browser component -------------------------------------------------
+
+
+def _browser_lines(machine) -> list[str]:
+    if not machine.pip_log.exists():
+        return []
+    return [line for line in machine.pip_log.read_text(encoding="utf-8").splitlines()
+            if "requirements-browser" in line or line.startswith("playwright ")]
+
+
+@pytest.mark.parametrize("shell", SHELLS)
+def test_without_with_browser_nothing_of_the_component_is_installed(machine, shell):
+    machine.standard()
+    (machine.tools / "requirements-browser.txt").write_text("playwright\n", encoding="utf-8")
+    result = machine.run(shell=shell)
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert _browser_lines(machine) == []
+    assert not any(line.split()[1:2] == ["browser"] for line in result.stdout.splitlines())
+
+
+@pytest.mark.parametrize("shell", SHELLS)
+def test_with_browser_installs_the_component_into_the_venv_and_keeps_it(machine, shell):
+    machine.standard()
+    (machine.tools / "requirements-browser.txt").write_text("playwright\n", encoding="utf-8")
+    result = machine.run("--with", "browser", shell=shell)
+    assert result.returncode == 0, result.stdout + result.stderr
+    venv = machine.tools / ".venv"
+    lines = _browser_lines(machine)
+    assert any("-r" in line and line.endswith("requirements-browser.txt") for line in lines)
+    assert f"playwright install chromium into {venv / 'ms-playwright'}" in lines
+    assert any(line.split()[:2] == ["OK", "browser"] for line in result.stdout.splitlines())
+    assert (venv / ".chemenu-browser.sha256").is_file()
+    # A later plain run keeps it, and reinstalls only when its requirements change.
+    assert machine.run(shell=shell).returncode == 0
+    assert len(_browser_lines(machine)) == 2
+    (machine.tools / "requirements-browser.txt").write_text("playwright>=2\n", encoding="utf-8")
+    result = machine.run(shell=shell)
+    assert result.returncode == 0 and len(_browser_lines(machine)) == 4
+    assert "OK       browser" in result.stdout
+
+
+@pytest.mark.parametrize("shell", SHELLS)
+@pytest.mark.parametrize("failing", ["FAKE_BROWSER_PIP_FAIL", "FAKE_PLAYWRIGHT_FAIL"])
+def test_a_component_that_fails_stops_but_leaves_wikitool_working(machine, shell, failing):
+    machine.standard()
+    (machine.tools / "requirements-browser.txt").write_text("playwright\n", encoding="utf-8")
+    machine.extra_env[failing] = "1"
+    result = machine.run("--with=browser", shell=shell)
+    assert result.returncode == 42
+    assert_guidance(result.stdout, "optional browser component", "--with browser")
+    assert machine.recorded()["complete"] is True
+    assert not (machine.tools / ".venv" / ".chemenu-browser.sha256").exists()
+
+
+@pytest.mark.parametrize("shell", SHELLS)
+def test_an_unknown_component_is_refused(machine, shell):
+    machine.standard()
+    result = machine.run("--with", "spreadsheet", shell=shell)
+    assert result.returncode == 1
+    assert "not an optional component" in result.stderr
+    assert not machine.tools_file.exists()
+
+
 # --- the store alias ----------------------------------------------------------------
 
 

tools/chemenu/tests/test_preflight_pwsh.py

@@ -183,6 +183,42 @@ def test_pip_failure_stops_and_is_retried_next_time(machine):
     assert machine.recorded()["complete"] is True
 
 
+def _browser_lines(machine: Machine) -> list[str]:
+    if not machine.pip_log.exists():
+        return []
+    return [line for line in machine.pip_log.read_text(encoding="utf-8").splitlines()
+            if "requirements-browser" in line or line.startswith("playwright ")]
+
+
+def test_with_browser_installs_the_component_and_a_plain_run_keeps_it(machine):
+    (machine.tools / "requirements-browser.txt").write_text("playwright\n", encoding="utf-8")
+    assert _preflight(machine).returncode == 0
+    assert _browser_lines(machine) == []
+    result = _preflight(machine, "--with", "browser")
+    assert result.returncode == 0, result.stdout + result.stderr
+    venv = machine.tools / ".venv"
+    assert f"playwright install chromium into {venv / 'ms-playwright'}" in _browser_lines(machine)
+    assert "OK       browser" in result.stdout
+    assert _preflight(machine).returncode == 0
+    assert len(_browser_lines(machine)) == 2
+
+
+@pytest.mark.parametrize("failing", ["FAKE_BROWSER_PIP_FAIL", "FAKE_PLAYWRIGHT_FAIL"])
+def test_a_component_that_fails_stops_but_leaves_wikitool_working(machine, failing):
+    (machine.tools / "requirements-browser.txt").write_text("playwright\n", encoding="utf-8")
+    machine.extra_env[failing] = "1"
+    result = _preflight(machine, "--with=browser")
+    assert result.returncode == 42
+    assert_guidance(result.stdout, "optional browser component", "--with browser")
+    assert machine.recorded()["complete"] is True
+
+
+def test_an_unknown_component_is_refused(machine):
+    result = _preflight(machine, "--with", "spreadsheet")
+    assert result.returncode == 1
+    assert "not an optional component" in result.stderr
+
+
 @pytest.mark.parametrize("platform, alias", [("linux", "python3"), ("windows", "python")])
 def test_store_alias_is_never_run_and_never_recorded(machine, platform, alias):
     marker = machine.base / "alias-was-run"

tools/chemenu/tests/test_raw_fetch_routes.py

@@ -0,0 +1,352 @@
+"""`wikitool raw fetch` along a route: with the user's session, through an
+external fetcher, or rendered by a browser (Gitea #169).
+
+As in `test_raw_fetch.py`, every request goes to a real `http.server` on
+127.0.0.1. The paid pages answer by the `Cookie` header they receive - the full
+article to the right session, a teaser to anything else - so what is under test
+is what a session actually sends, not what a stub was told. The browser is the
+one thing replaced: Playwright is an optional component that CI does not
+install, so `browser.render` and `browser.missing` are stood in for.
+"""
+from __future__ import annotations
+
+import datetime
+import json
+import os
+import sys
+from pathlib import Path
+
+import pytest
+import typer
+
+from chemenu import browser, config, config_shapes, fetch_routes, web_capture, web_session
+from chemenu.commands.raw.fetch import raw_fetch_command
+from chemenu.tests.conftest import write_settings
+from chemenu.tests.test_raw_fetch import ARTICLE, Site
+
+SECRET = "sid-6f3a-THIS-IS-THE-SESSION-VALUE"
+FULL = (
+    b"<!doctype html><html><head><meta charset=\"utf-8\"><title>Paid</title></head><body><main>"
+    b"<h1>The paid article</h1><p>All of the text, " + b"long enough to be an article. " * 10 + b"</p>"
+    b"<a class=\"logout\" href=\"/logout\">Log out</a></main></body></html>"
+)
+TEASER = (
+    b"<!doctype html><html><head><meta charset=\"utf-8\"><title>Paid</title></head><body><main>"
+    b"<h1>The paid article</h1><p>The first sentence.</p>"
+    b"<div class=\"paywall\">Subscribe to read on</div></main></body></html>"
+)
+HANDLER = {
+    "host": "127.0.0.1",
+    "summary": "A test publisher behind a login.",
+    "session": {
+        "login": "http://127.0.0.1/login",
+        "signed_in": {"tag": "a", "class": "logout"},
+        "signed_out": {"class": "paywall"},
+    },
+}
+
+
+def _state(value: str = SECRET, expires: float = -1) -> dict:
+    return {
+        "cookies": [{
+            "name": "sid", "value": value, "domain": "127.0.0.1", "path": "/", "expires": expires,
+            "httpOnly": True, "secure": False, "sameSite": "Lax",
+        }],
+        "origins": [],
+    }
+
+
+def _paid(server: Site, path: str = "/paid/article") -> str:
+    """A page that answers by the cookie it receives."""
+
+    def answer(handler) -> None:
+        handler.wfile.write(FULL if f"sid={SECRET}" in (handler.headers.get("Cookie") or "") else TEASER)
+
+    server.routes[path] = (200, {"Content-Type": "text/html; charset=utf-8"}, answer)
+    return server.url(path)
+
+
+@pytest.fixture
+def site(monkeypatch):
+    for var in ("http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"):
+        monkeypatch.delenv(var, raising=False)
+    server = Site()
+    yield server
+    server.stop()
+
+
+@pytest.fixture
+def tree(kb_dir, monkeypatch):
+    root = kb_dir.parent
+    (root / "raw").mkdir()
+    (root / "incoming").mkdir(exist_ok=True)
+    (root / "sites").mkdir()
+    return root
+
+
+def _handler(root: Path, **changes) -> None:
+    (root / "sites/publisher.json").write_text(json.dumps({**HANDLER, **changes}), encoding="utf-8")
+
+
+@pytest.fixture
+def routed(tree, monkeypatch):
+    """127.0.0.1 routed through the session, with a handler that names the login."""
+    _handler(tree)
+    write_settings(tree, monkeypatch, fetch={"routes": {"127.0.0.1": "session"}})
+    return tree
+
+
+def _session_env(monkeypatch, state: dict) -> None:
+    monkeypatch.setenv(config.ENV_SECRET_PREFIX + web_session.secret_name("127.0.0.1").upper(), json.dumps(state))
+
+
+def _fetch(url, name=None, scope=False):
+    return raw_fetch_command(url=url, html=None, source_url=None, name=name, scope=scope)
+
+
+def _exit_code(url, **kwargs) -> int:
+    with pytest.raises(typer.Exit) as excinfo:
+        _fetch(url, **kwargs)
+    return excinfo.value.exit_code
+
+
+def _header(md: Path) -> dict:
+    import yaml
+
+    content = md.read_text(encoding="utf-8")
+    return yaml.safe_load(content[4:].partition("\n---\n")[0])
+
+
+def _snapshot(root: Path) -> dict[str, bytes]:
+    return {str(p.relative_to(root)): p.read_bytes() for p in sorted(root.rglob("*")) if p.is_file()}
+
+
+# --- with the user's session ------------------------------------------------
+
+
+def test_with_a_stored_session_the_full_text_arrives_byte_identical(routed, site, monkeypatch, capsys):
+    _session_env(monkeypatch, _state())
+    url = _paid(site)
+    _fetch(url)
+    out = capsys.readouterr().out
+    assert (routed / "incoming/article.html").read_bytes() == FULL
+    header = _header(routed / "incoming/article.md")
+    assert header["session"] == "127.0.0.1"
+    assert header["fetched_by"] == "wikitool raw fetch"
+    assert "rendered" not in header
+    assert SECRET not in (routed / "incoming/article.md").read_text(encoding="utf-8")
+    assert SECRET not in out
+    assert "raw fetch --html" not in out  # the teaser hint is for the anonymous fetch
+
+
+def test_the_session_is_read_from_the_secrets_file_outside_the_tree(routed, site, monkeypatch):
+    path = web_session.store("127.0.0.1", "testinstance", _state())
+    assert path == config.secrets_file("testinstance")  # the user's configuration directory
+    if os.name != "nt":
+        assert path.stat().st_mode & 0o077 == 0
+    _fetch(_paid(site))
+    assert (routed / "incoming/article.html").read_bytes() == FULL
+    # The test's HOME lies inside its tmp_path; in it, the secrets file is the one place the value is.
+    holding = [p for p in routed.rglob("*") if p.is_file() and SECRET.encode() in p.read_bytes()]
+    assert holding == [path]
+
+
+@pytest.mark.parametrize("state", [None, _state("stale"), _state(expires=1_000_000_000)],
+                         ids=["no-session", "ended-by-the-website", "expired-cookie"])
+def test_without_a_valid_session_exit_42_names_raw_login_and_keeps_no_teaser(routed, site, monkeypatch, capsys, state):
+    if state is not None:
+        _session_env(monkeypatch, state)
+    url = _paid(site)
+    before = _snapshot(routed / "incoming")
+    assert _exit_code(url) == 42
+    out = capsys.readouterr().out
+    assert "tools/wikitool raw login 127.0.0.1" in out
+    assert _snapshot(routed / "incoming") == before
+    if state is None:
+        assert site.requests == []  # nothing to send, so nothing is asked
+
+
+def test_a_session_route_without_a_login_in_any_handler_is_refused_before_fetching(tree, site, monkeypatch, capsys):
+    (tree / "sites/publisher.json").write_text(
+        json.dumps({k: v for k, v in HANDLER.items() if k != "session"}), encoding="utf-8")
+    write_settings(tree, monkeypatch, fetch={"routes": {"127.0.0.1": "session"}})
+    assert _exit_code(_paid(site)) == 1
+    assert "names a login page" in capsys.readouterr().out
+    assert site.requests == []
+
+
+def test_scope_sends_the_session_with_every_page_and_names_it(routed, site, monkeypatch):
+    _handler(routed, path="/paid/", subpages={"links": {"tag": "a", "inside": "td"}})
+    write_settings(routed, monkeypatch, sites={"pause_seconds": 0})
+    _session_env(monkeypatch, _state())
+    index = (b"<html><body><main><h1>Index</h1><table><tr><td><a href=\"one\">One</a></td></tr></table>"
+             b"<a class=\"logout\" href=\"/logout\">Log out</a></main></body></html>")
+
+    def answer(handler) -> None:
+        handler.wfile.write(index if f"sid={SECRET}" in (handler.headers.get("Cookie") or "") else TEASER)
+
+    site.routes["/paid/"] = (200, {"Content-Type": "text/html"}, answer)
+    _paid(site, "/paid/one")
+    _fetch(site.url("/paid/"), name="bundle", scope=True)
+    bundle = routed / "incoming/bundle"
+    assert (bundle / "one.html").read_bytes() == FULL
+    assert _header(bundle / "one.md")["session"] == "127.0.0.1"
+    assert json.loads((bundle / "_scope.json").read_text(encoding="utf-8"))["session"] == "127.0.0.1"
+
+
+# --- rendered by a browser --------------------------------------------------
+
+
+def _fake_render(calls: list):
+    def render(url, user_agent, state=None, **kwargs):
+        calls.append((url, state))
+        return web_capture.Response(url, url, 200, "text/html; charset=utf-8", FULL)
+    return render
+
+
+def test_a_handler_declaring_render_gives_a_rendered_header(tree, site, monkeypatch):
+    _handler(tree, render=True)
+    calls: list = []
+    monkeypatch.setattr(browser, "render", _fake_render(calls))
+    url = site.html("/page", ARTICLE)
+    _fetch(url)
+    assert calls == [(url, None)]
+    assert site.requests == []  # the browser fetched it, not the built-in fetch
+    assert _header(tree / "incoming/page.md")["rendered"] is True
+
+
+def test_render_with_a_session_hands_the_browser_the_session(routed, site, monkeypatch):
+    _handler(routed, render=True)
+    _session_env(monkeypatch, _state())
+    calls: list = []
+    monkeypatch.setattr(browser, "render", _fake_render(calls))
+    _fetch(site.url("/paid/article"))
+    assert calls[0][1] == _state()
+    header = _header(routed / "incoming/article.md")
+    assert header["rendered"] is True and header["session"] == "127.0.0.1"
+
+
+def test_without_the_declaration_nothing_is_ever_rendered(routed, site, monkeypatch):
+    _session_env(monkeypatch, _state())
+
+    def refuse(*args, **kwargs):
+        raise AssertionError("rendered without a declaration")
+
+    monkeypatch.setattr(browser, "render", refuse)
+    _fetch(_paid(site))
+    assert "rendered" not in _header(routed / "incoming/article.md")
+
+
+def test_render_without_the_browser_component_exits_42_with_the_preflight_option(tree, site, monkeypatch, capsys):
+    _handler(tree, render=True)
+    monkeypatch.setattr(browser, "missing", lambda: "the Python package `playwright` is not installed in the venv")
+    before = _snapshot(tree / "incoming")
+    assert _exit_code(site.html("/page", ARTICLE)) == 42
+    assert "tools/preflight.sh --with browser" in capsys.readouterr().out
+    assert _snapshot(tree / "incoming") == before
+
+
+# --- an external fetcher ----------------------------------------------------
+
+
+def _fetcher(tmp_path: Path, body: bytes = FULL, code: int = 0, stderr: str = "") -> list[str]:
+    script = tmp_path / "fetcher.py"
+    script.write_text(
+        "import sys\n"
+        f"sys.stdout.buffer.write({body!r})\n"
+        f"sys.stderr.write({stderr!r})\n"
+        f"open({str(tmp_path / 'fetcher.args')!r}, 'w').write(' '.join(sys.argv[1:]))\n"
+        f"sys.exit({code})\n",
+        encoding="utf-8",
+    )
+    return [sys.executable, str(script), "--html"]
+
+
+def test_an_external_fetcher_writes_its_stdout_byte_identical(tree, site, monkeypatch, tmp_path):
+    write_settings(tree, monkeypatch, fetch={"routes": {"127.0.0.1": "homelab"},
+                                             "fetchers": {"homelab": _fetcher(tmp_path)}})
+    url = site.url("/elsewhere/post")
+    _fetch(url)
+    assert site.requests == []  # the built-in fetch never ran
+    assert (tmp_path / "fetcher.args").read_text(encoding="utf-8") == f"--html {url}"
+    assert (tree / "incoming/post.html").read_bytes() == FULL
+    header = _header(tree / "incoming/post.md")
+    assert header["fetched_by"] == "wikitool raw fetch (fetcher: homelab)"
+    assert "http_status" not in header
+
+
+def test_a_failing_fetcher_exits_1_with_its_stderr_and_no_fallback(tree, site, monkeypatch, tmp_path, capsys):
+    write_settings(tree, monkeypatch, fetch={
+        "routes": {"127.0.0.1": "homelab"},
+        "fetchers": {"homelab": _fetcher(tmp_path, code=3, stderr="upstream says 503")},
+    })
+    url = site.html("/post", ARTICLE)
+    before = _snapshot(tree / "incoming")
+    assert _exit_code(url) == 1
+    out = capsys.readouterr().out
+    assert "exit 3" in out and "upstream says 503" in out
+    assert _snapshot(tree / "incoming") == before
+    assert site.requests == []
+
+
+def test_a_route_to_a_fetcher_this_checkout_lacks_is_refused(tree, site, monkeypatch, capsys):
+    write_settings(tree, monkeypatch, fetch={"routes": {"127.0.0.1": "homelab"}})
+    assert _exit_code(site.html("/post", ARTICLE)) == 1
+    assert "fetch.fetchers.homelab" in capsys.readouterr().out
+    assert site.requests == []
+
+
+def test_a_fetcher_command_in_the_committed_file_is_refused(tree, site, monkeypatch, capsys, tmp_path):
+    (tree / config.INSTANCE_SETTINGS_FILENAME).write_text(json.dumps({
+        "schema": 1, "instance_id": "testinstance",
+        "fetch": {"routes": {"127.0.0.1": "homelab"}, "fetchers": {"homelab": _fetcher(tmp_path)}},
+    }), encoding="utf-8")
+    assert _exit_code(site.html("/post", ARTICLE)) == 1
+    out = capsys.readouterr().out
+    assert "fetch.fetchers" in out and "checkout" in out
+    assert site.requests == []
+    assert config_shapes.FETCH.field("fetchers").names_command
+    with pytest.raises(ValueError):
+        config_shapes.Field("fetchers", "object", "", tier=config_shapes.INSTANCE, names_command=True)
+
+
+# --- a host no route names --------------------------------------------------
+
+
+def _without_time(md: Path) -> str:
+    return "\n".join(line for line in md.read_text(encoding="utf-8").splitlines() if not line.startswith("retrieved:"))
+
+
+def test_a_host_no_route_names_is_fetched_exactly_as_before(tree, site, monkeypatch, capsys):
+    url = site.html("/post", ARTICLE)
+    _fetch(url, name="first")
+    first_out = capsys.readouterr().out
+    write_settings(tree, monkeypatch, fetch={"routes": {"other.example.org": "session"}})
+    _fetch(url, name="later")
+    later_out = capsys.readouterr().out
+    assert (tree / "incoming/later.html").read_bytes() == (tree / "incoming/first.html").read_bytes()
+    assert _without_time(tree / "incoming/later.md") == _without_time(tree / "incoming/first.md").replace(
+        "first.html", "later.html")
+    assert later_out.replace("later", "first") == first_out
+    assert [ua for _, ua in site.requests] == [site.requests[0][1]] * 2
+
+
+# --- the pieces -------------------------------------------------------------
+
+
+def test_secret_names_are_valid_reference_names():
+    assert web_session.secret_name("www.heise.de") == "session_www_heise_de"
+    assert web_session.secret_name("a--b.example") == "session_a_b_example"
+
+
+def test_expired_cookies_are_not_sent(site):
+    jar = web_session.cookie_jar(_state(expires=(datetime.datetime.now() - datetime.timedelta(days=1)).timestamp()))
+    url = _paid(site)
+    response = web_capture.fetch(url, "test", cookies=jar)
+    assert response.body == TEASER
+
+
+def test_a_route_names_a_session_label_only_for_the_session():
+    assert fetch_routes.Route("h", fetch_routes.SESSION).session_label == "h"
+    assert fetch_routes.Route("h", fetch_routes.FETCHER, fetcher="x").session_label == ""
+    assert fetch_routes.Route("h", fetch_routes.FETCHER, fetcher="x").fetched_by.endswith("(fetcher: x)")

tools/chemenu/tests/test_raw_login.py

@@ -0,0 +1,124 @@
+"""`wikitool raw login <host>` - the user signs in once, and the session is stored
+as a secret outside the working tree (Gitea #169).
+
+The browser window itself needs a human and the optional Playwright component,
+so `browser.login` and `browser.missing` are stood in for: what is under test is
+the order of the checks, where the session lands, and that nothing prints it.
+"""
+from __future__ import annotations
+
+import json
+import os
+
+import pytest
+import typer
+
+from chemenu import browser, config, web_session
+from chemenu.commands.raw.fetch import raw_fetch_command
+from chemenu.commands.raw.login import raw_login_command
+from chemenu.tests.conftest import write_settings
+from chemenu.tests.test_raw_fetch_routes import FULL, HANDLER, SECRET, _paid, _state, site, tree  # noqa: F401
+
+
+@pytest.fixture
+def routed(tree, monkeypatch):  # noqa: F811
+    (tree / "sites/publisher.json").write_text(json.dumps(HANDLER), encoding="utf-8")
+    write_settings(tree, monkeypatch, fetch={"routes": {"127.0.0.1": "session"}})
+    return tree
+
+
+def _login(host: str = "127.0.0.1") -> int:
+    try:
+        raw_login_command(host=host)
+    except typer.Exit as exc:
+        return exc.exit_code
+    return 0
+
+
+def test_login_stores_the_session_outside_the_tree_and_prints_none_of_it(routed, site, monkeypatch, capsys):  # noqa: F811
+    opened = []
+    monkeypatch.setattr(browser, "login", lambda url: opened.append(url) or _state())
+    assert _login() == 0
+    out = capsys.readouterr()
+    assert opened == ["http://127.0.0.1/login"]
+    path = config.secrets_file("testinstance")
+    stored = json.loads(json.loads(path.read_text(encoding="utf-8"))["session_127_0_0_1"])
+    assert stored == _state()
+    if os.name != "nt":
+        assert path.stat().st_mode & 0o077 == 0
+    assert SECRET not in out.out + out.err
+    assert "session_127_0_0_1" in out.out
+    # ... and the next fetch uses it.
+    raw_fetch_command(url=_paid(site), html=None, source_url=None, name=None, scope=False)
+    assert (routed / "incoming/article.html").read_bytes() == FULL
+
+
+def test_login_keeps_the_other_secrets(routed, monkeypatch):
+    path = config.secrets_file("testinstance")
+    path.parent.mkdir(parents=True)
+    path.write_text(json.dumps({"caldav_app_password": "other"}), encoding="utf-8")
+    path.chmod(0o600)
+    monkeypatch.setattr(browser, "login", lambda url: _state())
+    assert _login() == 0
+    assert json.loads(path.read_text(encoding="utf-8"))["caldav_app_password"] == "other"
+
+
+def test_login_takes_a_url_for_its_host(routed, monkeypatch):
+    monkeypatch.setattr(browser, "login", lambda url: _state())
+    assert _login("http://127.0.0.1/some/article") == 0
+    assert web_session.load("127.0.0.1", "testinstance") == _state()
+
+
+def test_login_without_the_browser_component_exits_42_with_the_preflight_option(routed, monkeypatch, capsys):
+    monkeypatch.setattr(browser, "missing", lambda: "the Python package `playwright` is not installed in the venv")
+    assert _login() == 42
+    out = capsys.readouterr().out
+    assert "tools/preflight.sh --with browser" in out
+    assert not config.secrets_file("testinstance").exists()
+
+
+def test_login_for_a_host_not_routed_through_the_session_is_refused(tree, monkeypatch, capsys):  # noqa: F811
+    (tree / "sites/publisher.json").write_text(json.dumps(HANDLER), encoding="utf-8")
+    monkeypatch.setattr(browser, "login", lambda url: pytest.fail("the browser was opened"))
+    assert _login() == 1
+    assert "config set fetch.routes" in capsys.readouterr().out
+
+
+def test_login_for_a_host_whose_handler_names_no_login_page_is_refused(tree, monkeypatch, capsys):  # noqa: F811
+    write_settings(tree, monkeypatch, fetch={"routes": {"127.0.0.1": "session"}})
+    monkeypatch.setattr(browser, "login", lambda url: pytest.fail("the browser was opened"))
+    assert _login() == 1
+    assert "names a login page" in capsys.readouterr().out
+
+
+def test_login_warns_when_a_variable_answers_before_the_file(routed, monkeypatch, capsys):
+    monkeypatch.setenv("WIKITOOL_SECRET_SESSION_127_0_0_1", json.dumps(_state("older")))
+    monkeypatch.setattr(browser, "login", lambda url: _state())
+    assert _login() == 0
+    assert "WARN WIKITOOL_SECRET_SESSION_127_0_0_1 is set" in capsys.readouterr().out
+
+
+def test_the_component_is_reported_missing_without_playwright(monkeypatch):
+    monkeypatch.setattr(browser.importlib.util, "find_spec", lambda name: None)
+    assert "playwright" in browser.missing()
+    with pytest.raises(browser.BrowserMissing, match="--with browser"):
+        browser.require("`raw login`")
+
+
+def test_config_set_writes_a_route_and_leaves_a_fetcher_to_the_user(tree):  # noqa: F811
+    from typer.testing import CliRunner
+
+    from chemenu.cli import app
+
+    runner = CliRunner()
+    result = runner.invoke(app, ["config", "set", "fetch.routes", '{"www.example.org": "session"}'])
+    assert result.exit_code == 0, result.output
+    data = json.loads((tree / config.INSTANCE_SETTINGS_FILENAME).read_text(encoding="utf-8"))
+    assert data["fetch"] == {"routes": {"www.example.org": "session"}} and data["instance_id"]
+
+    result = runner.invoke(app, ["config", "set", "fetch.fetchers", '{"homelab": ["/bin/true"]}'])
+    assert result.exit_code == 42
+    assert not (tree / config.LOCAL_SETTINGS_FILENAME).exists()
+
+    result = runner.invoke(app, ["config", "set", "fetch.routes", '{"www.example.org": "Session!"}'])
+    assert result.exit_code == 1

tools/chemenu/tests/test_site_files.py

@@ -190,6 +190,10 @@ BROKEN = {
     "a placeholder without its field": {**EXAMPLE, "checks": [{"text": "see {count}"}]},
     "a parent that does not exist": {**EXAMPLE, "parent": "nobody"},
     "a parent for another host": {**EXAMPLE, "parent": "learn-microsoft"},
+    "a session without a sign of being signed in": {**EXAMPLE, "session": {"login": "https://docs.example.org/login"}},
+    "a session whose login page is not http(s)": {
+        **EXAMPLE, "session": {"login": "file:///etc/passwd", "signed_in": {"class": "logout"}}},
+    "a render that is not a boolean": {**EXAMPLE, "render": "yes"},
 }
 
 
@@ -273,6 +277,20 @@ def test_a_symlink_is_not_followed(tree):
     assert doctor.check_site_handlers().status == "FAIL"
 
 
+def test_a_session_and_render_are_read_and_inherited(tree):
+    _handler(tree, "example", {**EXAMPLE, "render": True, "session": {
+        "login": "https://docs.example.org/login", "signed_out": {"class": "paywall"}}})
+    _handler(tree, "example-guide", {"host": "docs.example.org", "summary": "The guide.", "path": "/guide/",
+                                     "parent": "example"})
+    instance, broken = sites.read_instance(tree)
+    assert broken == ()
+    child = next(s for s in instance if s.name == "example-guide")
+    assert child.render and child.session.login == "https://docs.example.org/login"
+    assert child.session.is_signed_in("<p>article</p>")
+    assert not child.session.is_signed_in('<div class="paywall">subscribe</div>')
+    assert sites.login_site("docs.example.org", instance=instance).name == "example"
+
+
 def test_doctor_is_quiet_without_a_sites_directory(tree):
     assert doctor.check_site_handlers().status == "OK"