sites: the instance writes its own site handlers as JSON files under sites/, the stack's learn.microsoft.com handlers become data files in the same format (#210); 10.0.0-beta.4

Files changed:
- .gitignore
- AGENTS.md
- CHANGES.md
- VERSION
- docs/explanation/architecture.md
- docs/how-to/README.md
- docs/how-to/write-a-site-handler.md
- docs/reference/commands/instance-health.md
- docs/reference/commands/raw-material-and-uploads.md
- docs/reference/configuration.md
- instructions/dev/doc-pull-through.md
- raw/CONTRACT.md
- sites/CONTRACT.md
- tools/CONTRACT.md
- tools/README.md
- tools/chemenu/commands/dist_cmd.py
- tools/chemenu/commands/docs_verify.py
- tools/chemenu/commands/doctor.py
- tools/chemenu/commands/git_publish.py
- tools/chemenu/commands/raw/fetch.py
- tools/chemenu/ownership.py
- tools/chemenu/sites/__init__.py
- tools/chemenu/sites/data.py
- tools/chemenu/sites/handlers/learn-microsoft-powershell-module.json
- tools/chemenu/sites/handlers/learn-microsoft.json
- tools/chemenu/sites/learn_microsoft.py
- tools/chemenu/sites/site.schema.json
- tools/chemenu/tests/test_dist_cmd.py
- tools/chemenu/tests/test_doctor.py
- tools/chemenu/tests/test_site_files.py
- tools/chemenu/toc.py

Co-Authored-By: Claude Code <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2

tools/chemenu/tests/test_dist_cmd.py

@@ -155,7 +155,7 @@ def repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
     (docs_dir / "dev").mkdir()
     (docs_dir / "dev" / "development.md").write_text("# Developing this stack\n", encoding="utf-8")
 
-    for relative in ("raw/CONTRACT.md", "reports/CONTRACT.md", "work/CONTRACT.md"):
+    for relative in ("raw/CONTRACT.md", "reports/CONTRACT.md", "work/CONTRACT.md", "sites/CONTRACT.md"):
         path = root / relative
         path.parent.mkdir(parents=True, exist_ok=True)
         path.write_text(f"# {relative}\n", encoding="utf-8")
@@ -405,6 +405,7 @@ def test_find_leaks_is_silent_on_a_clean_plan(repo):
         "instructions/dev/commonplace-kb.md",
         "kb/entities/aurora.md",
         "raw/notes/personal-note.md",
+        "sites/example.json",
         # Both bind every page and both are the instance's to write, so the
         # filled name must never cross - only the `.template` beside it does.
         "kb/CONVENTIONS.md",
@@ -453,6 +454,14 @@ def test_plan_ships_collection_contracts_as_templates_not_pages(repo):
     assert plan["kb/entities/COLLECTION.md.template"].content == "# entities collection\n"
 
 
+def test_plan_ships_the_sites_contract_and_no_handler_file_of_the_instance(repo):
+    (repo / "sites").mkdir(exist_ok=True)
+    (repo / "sites" / "example.json").write_text("{}", encoding="utf-8")
+    plan = dist_cmd.build_plan()
+    assert "sites/CONTRACT.md" in plan
+    assert [relative for relative in plan if relative.startswith("sites/")] == ["sites/CONTRACT.md"]
+
+
 def test_plan_ships_the_conventions_template_and_not_the_filled_file(repo):
     plan = dist_cmd.build_plan()
     assert "kb/CONVENTIONS.md.template" in plan

tools/chemenu/tests/test_doctor.py

@@ -61,7 +61,7 @@ def instance(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
     (root / "SOUL.md").write_text("# SOUL.md - Fixture\n", encoding="utf-8")
 
     for relative in ("raw/CONTRACT.md", "reports/CONTRACT.md", "work/CONTRACT.md",
-                      "instructions/CONTRACT.md", "types/type-spec.md"):
+                      "sites/CONTRACT.md", "instructions/CONTRACT.md", "types/type-spec.md"):
         path = root / relative
         path.parent.mkdir(parents=True, exist_ok=True)
         path.write_text("# contract\n", encoding="utf-8")

tools/chemenu/tests/test_site_files.py

@@ -0,0 +1,289 @@
+"""The instance's own site handlers: JSON files in `sites/` (`chemenu.sites.data`).
+
+The stack's handlers are the same kind of file, so the checks of `test_sites.py`
+over the Learn handlers cover the format itself; this module is about what is
+particular to a file the instance wrote - that it takes effect, that it can replace
+a stack handler, and that one that cannot be used stops the fetch instead of being
+skipped.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+import typer
+import yaml
+
+from chemenu import sites, web_capture
+from chemenu.commands import doctor
+from chemenu.commands.raw.fetch import raw_fetch_command
+from chemenu.errors import ValidationError
+from chemenu.tests.conftest import write_settings
+
+PAGE = (Path(__file__).parent / "fixtures/learn_microsoft/oslicense-module.html").read_bytes()
+
+EXAMPLE = {
+    "host": "docs.example.org",
+    "summary": "Example docs: drops the action bar and says the source is elsewhere.",
+    "drop": [{"id": "action-panel"}, {"attr": "data-list-type", "value": "plan"}],
+    "checks": [{"text": "The page keeps its source at {meta}.", "meta": "original_content_git_url"}],
+}
+
+
+@pytest.fixture
+def tree(kb_dir):
+    root = kb_dir.parent
+    (root / "raw").mkdir()
+    (root / "incoming").mkdir(exist_ok=True)
+    return root
+
+
+@pytest.fixture
+def served(monkeypatch):
+    asked: list[str] = []
+
+    def fake(url, user_agent, timeout=0, max_bytes=0):
+        asked.append(url)
+        return web_capture.Response(url, url, 200, "text/html; charset=utf-8", PAGE)
+
+    monkeypatch.setattr(web_capture, "fetch", fake)
+    return asked
+
+
+def _handler(root: Path, name: str, content) -> Path:
+    directory = root / "sites"
+    directory.mkdir(exist_ok=True)
+    path = directory / f"{name}.json"
+    path.write_text(content if isinstance(content, str) else json.dumps(content), encoding="utf-8")
+    return path
+
+
+def _fetch(url, name=None, html=None):
+    return raw_fetch_command(url=None if html else url, html=html, source_url=url if html else None, name=name)
+
+
+def _refused(url, **kwargs) -> None:
+    with pytest.raises(typer.Exit) as excinfo:
+        _fetch(url, **kwargs)
+    assert excinfo.value.exit_code == 1
+
+
+def _md(path: Path) -> tuple[dict, str]:
+    head, _, text = path.read_text(encoding="utf-8")[4:].partition("\n---\n")
+    return yaml.safe_load(head), text
+
+
+# --- a handler of the instance takes effect ----------------------------------
+
+
+def test_an_instance_handler_cleans_the_page_and_says_so_in_the_header(tree, served, capsys):
+    _handler(tree, "example", EXAMPLE)
+    _fetch("https://docs.example.org/guide", name="guide")
+    header, text = _md(tree / "incoming/guide.md")
+
+    assert header["site"] == "example (instance)"
+    assert "Add to Plans" not in text and "Exit editor mode" in text
+    checks = [line for line in capsys.readouterr().out.splitlines() if line.startswith("CHECK ")]
+    assert len(checks) == 1 and "docs-source/blob/live" in checks[0]
+
+
+def test_an_instance_handler_rewrites_the_url_like_a_stack_handler(tree, served):
+    _handler(tree, "example", {
+        **EXAMPLE,
+        "rewrites": [{"pattern": r"^(https://docs\.example\.org/)[a-z]{2}/", "replacement": r"\1{locale}/"}],
+    })
+    _fetch("https://docs.example.org/de/guide", name="guide")
+    header, _ = _md(tree / "incoming/guide.md")
+
+    assert served == ["https://docs.example.org/en-us/guide"]
+    assert header["requested_url"] == "https://docs.example.org/de/guide"
+
+
+def test_a_handler_for_another_host_does_not_touch_the_page(tree, served):
+    _handler(tree, "example", EXAMPLE)
+    _fetch("https://example.org/guide", name="guide")
+    header, text = _md(tree / "incoming/guide.md")
+
+    assert "site" not in header and "Add to Plans" in text
+
+
+def test_the_handler_applies_to_a_saved_page_too(tree, monkeypatch):
+    monkeypatch.setattr(web_capture, "fetch", lambda *a, **k: pytest.fail("no network"))
+    _handler(tree, "example", EXAMPLE)
+    saved = tree / "incoming/guide.html"
+    saved.write_bytes(PAGE)
+    _fetch("https://docs.example.org/guide", html=saved)
+
+    header, text = _md(tree / "incoming/guide.md")
+    assert header["site"] == "example (instance)" and "Add to Plans" not in text
+
+
+def test_an_instance_handler_may_inherit_from_a_stack_handler(tree, served):
+    _handler(tree, "learn-extra", {
+        "host": "learn.microsoft.com",
+        "summary": "Also drops the plan button.",
+        "parent": "learn-microsoft",
+        "path": "/en-us/azure/",
+        "drop": [{"attr": "data-list-type", "value": "plan"}],
+    })
+    _fetch("https://learn.microsoft.com/en-us/azure/overview", name="azure")
+    header, text = _md(tree / "incoming/azure.md")
+
+    assert header["site"] == "learn-extra (instance)"
+    assert "Add to Plans" not in text and "Exit editor mode" not in text
+
+
+def test_without_a_sites_directory_nothing_changes(tree):
+    preferences = sites.read_preferences(tree)
+    assert preferences.instance == () and preferences.broken == ()
+
+
+# --- replacing a stack handler -----------------------------------------------
+
+
+def test_a_handler_for_the_same_host_and_path_replaces_the_stack_handler(tree, served):
+    _handler(tree, "mine", {"host": "learn.microsoft.com", "summary": "Keeps everything.", "drop": [{"id": "action-panel"}]})
+    _fetch("https://learn.microsoft.com/en-us/azure/overview", name="azure")
+    header, text = _md(tree / "incoming/azure.md")
+
+    assert header["site"] == "mine (instance)"
+    assert "Exit editor mode" in text
+    assert "learn-microsoft" not in [s.name for s in sites.catalogue(sites.read_preferences(tree).instance)]
+
+
+def test_the_stack_handlers_that_inherited_from_a_replaced_one_keep_what_they_inherited(tree, served):
+    _handler(tree, "mine", {"host": "learn.microsoft.com", "summary": "Replaces the general one."})
+    _fetch("https://learn.microsoft.com/de-de/powershell/module/oslicense/", name="module")
+    header, text = _md(tree / "incoming/module.md")
+
+    assert header["site"] == "learn-microsoft-powershell-module"
+    assert "Exit editor mode" not in text
+
+
+def test_doctor_lists_the_replacement(tree):
+    _handler(tree, "mine", {"host": "learn.microsoft.com", "summary": "Replaces the general one."})
+    check = doctor.check_site_handlers()
+
+    assert check.status == "OK"
+    assert "mine (replaces the stack's learn-microsoft)" in check.detail
+
+
+def test_a_replaced_stack_handler_can_no_longer_be_disabled_by_its_name(tree, monkeypatch):
+    _handler(tree, "mine", {"host": "learn.microsoft.com", "summary": "Replaces the general one."})
+    write_settings(tree, monkeypatch, sites={"disabled": ["learn-microsoft"]})
+    with pytest.raises(ValidationError, match="no handler has"):
+        sites.read_preferences(tree)
+
+
+# --- a file that cannot be used ----------------------------------------------
+
+BROKEN = {
+    "unknown field": {**EXAMPLE, "colour": "red"},
+    "missing summary": {"host": "docs.example.org"},
+    "a rule that matches everything": {**EXAMPLE, "drop": [{"inside": "td"}]},
+    "a value without an attr": {**EXAMPLE, "drop": [{"tag": "a", "value": "x"}]},
+    "more than three checks": {**EXAMPLE, "checks": [{"text": "a"}] * 4},
+    "a pattern that does not compile": {**EXAMPLE, "rewrites": [{"pattern": "(", "replacement": "x"}]},
+    "a replacement naming a group that is not there": {**EXAMPLE, "rewrites": [{"pattern": "a", "replacement": r"\1"}]},
+    "an upper-case host": {**EXAMPLE, "host": "Docs.Example.org"},
+    "a placeholder without its field": {**EXAMPLE, "checks": [{"text": "see {count}"}]},
+    "a parent that does not exist": {**EXAMPLE, "parent": "nobody"},
+    "a parent for another host": {**EXAMPLE, "parent": "learn-microsoft"},
+}
+
+
+@pytest.mark.parametrize("case", sorted(BROKEN))
+def test_a_file_that_breaks_a_rule_fails_doctor_and_stops_a_fetch_of_its_host(tree, served, case):
+    _handler(tree, "example", BROKEN[case])
+    host = BROKEN[case]["host"].lower()
+
+    check = doctor.check_site_handlers()
+    assert check.status == "FAIL" and "sites/example.json" in check.detail
+
+    _refused(f"https://{host}/guide", name="guide")
+    assert served == [] and list((tree / "incoming").iterdir()) == []
+
+
+def test_a_stack_name_for_another_host_is_refused(tree, served):
+    _handler(tree, "learn-microsoft", {"host": "other.example.org", "summary": "x"})
+    check = doctor.check_site_handlers()
+
+    assert check.status == "FAIL" and "another host or path" in check.detail
+    _refused("https://other.example.org/guide", name="guide")
+    assert served == []
+
+
+def test_a_file_that_is_not_json_stops_every_fetch(tree, served):
+    _handler(tree, "example", "{not json")
+    assert doctor.check_site_handlers().status == "FAIL"
+    _refused("https://example.org/guide", name="guide")
+    assert served == [] and list((tree / "incoming").iterdir()) == []
+
+
+def test_a_broken_file_for_another_host_does_not_stop_a_fetch(tree, served):
+    _handler(tree, "example", {**EXAMPLE, "colour": "red"})
+    _fetch("https://example.org/guide", name="guide")
+    assert served == ["https://example.org/guide"]
+
+
+def test_a_saved_page_is_refused_the_same_way(tree):
+    _handler(tree, "example", {**EXAMPLE, "colour": "red"})
+    saved = tree / "incoming/guide.html"
+    saved.write_bytes(PAGE)
+    _refused("https://docs.example.org/guide", html=saved)
+    assert not (tree / "incoming/guide.md").exists()
+
+
+def test_a_file_whose_name_is_not_a_handler_name_is_broken(tree):
+    _handler(tree, "My Handler", EXAMPLE)
+    assert doctor.check_site_handlers().status == "FAIL"
+
+
+def test_two_files_for_the_same_host_and_path_are_an_error(tree):
+    _handler(tree, "a", EXAMPLE)
+    _handler(tree, "b", EXAMPLE)
+    check = doctor.check_site_handlers()
+
+    assert check.status == "FAIL" and "sites/b.json" in check.detail and "same host and path" in check.detail
+    assert [s.name for s in sites.read_instance(tree)[0]] == ["a"]
+
+
+def test_parents_that_form_a_cycle_are_broken(tree):
+    _handler(tree, "a", {**EXAMPLE, "path": "/a", "parent": "b"})
+    _handler(tree, "b", {**EXAMPLE, "path": "/b", "parent": "a"})
+    instance, broken = sites.read_instance(tree)
+
+    assert instance == () and {b.file for b in broken} == {"sites/a.json", "sites/b.json"}
+
+
+def test_a_child_of_a_broken_parent_is_broken_too(tree):
+    _handler(tree, "base", {**EXAMPLE, "colour": "red"})
+    _handler(tree, "child", {**EXAMPLE, "path": "/x", "parent": "base"})
+    instance, broken = sites.read_instance(tree)
+
+    assert instance == () and {b.file for b in broken} == {"sites/base.json", "sites/child.json"}
+
+
+def test_a_symlink_is_not_followed(tree):
+    target = tree / "elsewhere.json"
+    target.write_text(json.dumps(EXAMPLE), encoding="utf-8")
+    (tree / "sites").mkdir()
+    (tree / "sites" / "example.json").symlink_to(target)
+    assert doctor.check_site_handlers().status == "FAIL"
+
+
+def test_doctor_is_quiet_without_a_sites_directory(tree):
+    assert doctor.check_site_handlers().status == "OK"
+
+
+# --- the stack's handlers are files, too -------------------------------------
+
+
+def test_the_stack_handlers_are_data_files_that_pass_their_own_schema():
+    from chemenu.sites import data
+
+    names = {path.stem for path in data.STACK_DIRECTORY.glob("*.json")}
+    assert {"learn-microsoft", "learn-microsoft-powershell-module"} <= names
+    assert {s.name for s in sites.all_sites()} >= names
+    assert all(s.origin == "stack" for s in sites.all_sites())