sites: raw fetch --scope fetches a page and the subpages its site handler offers as one bundle in incoming/<stem>/, with robots.txt, max_pages and pause_seconds; 10.0.0-beta.6
Files changed: - CHANGES.md - VERSION - docs/how-to/ingest-a-source.md - docs/how-to/write-a-site-handler.md - docs/reference/commands/raw-material-and-uploads.md - docs/reference/configuration.md - raw/CONTRACT.md - sites/CONTRACT.md - tools/CONTRACT.md - tools/README.md - tools/chemenu/commands/raw/fetch.py - tools/chemenu/config_shapes.py - tools/chemenu/sites/__init__.py - tools/chemenu/sites/data.py - tools/chemenu/sites/handlers/learn-microsoft-powershell-module.json - tools/chemenu/sites/site.schema.json - tools/chemenu/tests/test_raw_fetch.py - tools/chemenu/tests/test_raw_fetch_scope.py - tools/chemenu/tests/test_site_files.py - tools/chemenu/tests/test_sites.py - tools/chemenu/web_capture.py Co-Authored-By: Claude Code <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
This commit is contained in:
torbenandClaude Code committed
1 parent
f25ee6a1ce
commit
21cdd787b7
21 files changed
+835
-41
No files matched your search
@@ -100,8 +100,8 @@ def tree(kb_dir):
|
||||
return root
|
||||
|
||||
|
||||
def _fetch(url=None, html=None, source_url=None, name=None):
|
||||
return raw_fetch_command(url=url, html=html, source_url=source_url, name=name)
|
||||
def _fetch(url=None, html=None, source_url=None, name=None, scope=False):
|
||||
return raw_fetch_command(url=url, html=html, source_url=source_url, name=name, scope=scope)
|
||||
|
||||
|
||||
def _refused(**kwargs) -> None:
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
"""`wikitool raw fetch --scope` - a page and the subpages its site handler offers,
|
||||
fetched as one bundle.
|
||||
|
||||
Like `test_raw_fetch.py`, every request goes to a real `http.server` on 127.0.0.1.
|
||||
The handler is an instance handler in `sites/`, because that is the form in which
|
||||
the subpages building block is data an instance can write.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import typer
|
||||
|
||||
from chemenu import sites, web_capture
|
||||
from chemenu.commands.raw.accept import raw_accept_command
|
||||
from chemenu.commands.raw.fetch import raw_fetch_command
|
||||
from chemenu.config_shapes import SITES, check
|
||||
from chemenu.tests.conftest import write_settings
|
||||
from chemenu.tests.test_raw_fetch import Site
|
||||
|
||||
HANDLER = {
|
||||
"host": "127.0.0.1",
|
||||
"summary": "A test module: the links in its table are its subpages.",
|
||||
"path": "/mod/",
|
||||
"subpages": {"links": {"tag": "a", "inside": "td"}},
|
||||
}
|
||||
|
||||
|
||||
def _index(port: int) -> bytes:
|
||||
return (
|
||||
"<html><head><title>Module</title></head><body><main><h1>Module</h1><table>"
|
||||
'<tr><td><a href="a?v=1#top">A</a></td></tr>'
|
||||
'<tr><td><a href="b?v=1">B</a></td></tr>'
|
||||
'<tr><td><a href="/mod/c">C</a></td></tr>'
|
||||
'<tr><td><a href="a?v=1">A again</a></td></tr>'
|
||||
'<tr><td><a href="/elsewhere/x">outside the prefix</a></td></tr>'
|
||||
f'<tr><td><a href="http://localhost:{port}/mod/foreign">another host</a></td></tr>'
|
||||
'<tr><td><a href="mailto:x@example.org">mail</a></td></tr>'
|
||||
'<tr><td><a href="./">the page itself</a></td></tr>'
|
||||
"</table>"
|
||||
'<p><a href="/mod/hidden">a link outside the table</a></p>'
|
||||
"</main></body></html>"
|
||||
).encode()
|
||||
|
||||
|
||||
def _sub(name: str) -> bytes:
|
||||
return (
|
||||
f"<html><head><title>{name}</title></head><body><main><h1>{name}</h1>"
|
||||
f"<p>{name} text</p></main></body></html>"
|
||||
).encode()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def site(monkeypatch):
|
||||
for var in ("http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
server = Site()
|
||||
yield server
|
||||
server.stop()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tree(kb_dir, monkeypatch, site):
|
||||
root = kb_dir.parent
|
||||
(root / "raw").mkdir()
|
||||
(root / "incoming").mkdir(exist_ok=True)
|
||||
(root / "sites").mkdir()
|
||||
(root / "sites/test-module.json").write_text(json.dumps(HANDLER), encoding="utf-8")
|
||||
write_settings(root, monkeypatch, sites={"pause_seconds": 0})
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def module(site):
|
||||
"""The index and the pages around it, served."""
|
||||
site.html("/mod/", _index(site.httpd.server_address[1]))
|
||||
for name in ("a?v=1", "b?v=1", "c", "hidden", "foreign"):
|
||||
site.html(f"/mod/{name}", _sub(name.split("?")[0]))
|
||||
return site
|
||||
|
||||
|
||||
def _fetch(url, name=None, scope=True):
|
||||
return raw_fetch_command(url=url, html=None, source_url=None, name=name, scope=scope)
|
||||
|
||||
|
||||
def _refused(url, **kwargs) -> None:
|
||||
with pytest.raises(typer.Exit) as excinfo:
|
||||
_fetch(url, **kwargs)
|
||||
assert excinfo.value.exit_code == 1
|
||||
|
||||
|
||||
def _snapshot(root: Path) -> list[str]:
|
||||
return sorted(str(p.relative_to(root)) for p in root.rglob("*"))
|
||||
|
||||
|
||||
def _requested(site: Site) -> list[str]:
|
||||
return [path for path, _ in site.requests]
|
||||
|
||||
|
||||
def _checks(capsys) -> list[str]:
|
||||
return [line[len("CHECK "):] for line in capsys.readouterr().out.splitlines() if line.startswith("CHECK ")]
|
||||
|
||||
|
||||
# --- without --scope --------------------------------------------------------
|
||||
|
||||
|
||||
def test_without_scope_the_page_is_fetched_alone_and_the_offer_is_one_check(tree, module, capsys):
|
||||
_fetch(module.url("/mod/"), name="mod", scope=False)
|
||||
|
||||
assert sorted(p.name for p in (tree / "incoming").iterdir()) == ["mod.html", "mod.md"]
|
||||
assert (tree / "incoming/mod.html").read_bytes() == module.routes["/mod/"][2]
|
||||
assert _requested(module) == ["/mod/"]
|
||||
lines = _checks(capsys)
|
||||
assert len(lines) == 1
|
||||
assert "3 subpage(s)" in lines[0] and "--scope" in lines[0]
|
||||
|
||||
|
||||
# --- with --scope -----------------------------------------------------------
|
||||
|
||||
|
||||
def test_scope_writes_a_folder_with_a_pair_per_page_and_the_manifest(tree, module):
|
||||
before = datetime.datetime.now(datetime.timezone.utc)
|
||||
_fetch(module.url("/mod/"), name="mod")
|
||||
|
||||
folder = tree / "incoming/mod"
|
||||
assert sorted(p.name for p in folder.iterdir()) == [
|
||||
"_scope.json", "a.html", "a.md", "b.html", "b.md", "c.html", "c.md", "mod.html", "mod.md",
|
||||
]
|
||||
assert [p.name for p in (tree / "incoming").iterdir()] == ["mod"]
|
||||
for name, path in (("mod", "/mod/"), ("a", "/mod/a?v=1"), ("b", "/mod/b?v=1"), ("c", "/mod/c")):
|
||||
assert (folder / f"{name}.html").read_bytes() == module.routes[path][2]
|
||||
text = (folder / "a.md").read_text(encoding="utf-8")
|
||||
assert "a text" in text and "derived_from: a.html" in text
|
||||
|
||||
manifest = json.loads((folder / "_scope.json").read_text(encoding="utf-8"))
|
||||
assert manifest["scope"] == 1 and manifest["url"] == module.url("/mod/")
|
||||
assert [(p["name"], p["role"]) for p in manifest["pages"]] == [
|
||||
("mod", "index"), ("a", "subpage"), ("b", "subpage"), ("c", "subpage"),
|
||||
]
|
||||
for page in manifest["pages"]:
|
||||
html = (folder / f"{page['name']}.html").read_bytes()
|
||||
assert page["sha256"] == hashlib.sha256(html).hexdigest()
|
||||
assert page["files"] == [f"{page['name']}.html", f"{page['name']}.md"]
|
||||
stamp = datetime.datetime.strptime(page["retrieved"], "%Y-%m-%dT%H:%M:%SZ").replace(
|
||||
tzinfo=datetime.timezone.utc)
|
||||
assert before - datetime.timedelta(seconds=2) <= stamp <= datetime.datetime.now(datetime.timezone.utc)
|
||||
assert [p["url"] for p in manifest["pages"][1:]] == [
|
||||
module.url("/mod/a?v=1"), module.url("/mod/b?v=1"), module.url("/mod/c"),
|
||||
]
|
||||
|
||||
|
||||
def test_only_the_urls_the_handler_names_are_requested(tree, module):
|
||||
_fetch(module.url("/mod/"), name="mod")
|
||||
|
||||
paths = _requested(module)
|
||||
assert sorted(paths) == sorted(["/mod/", "/robots.txt", "/mod/a?v=1", "/mod/b?v=1", "/mod/c"])
|
||||
assert not {"/mod/hidden", "/mod/foreign", "/elsewhere/x"} & set(paths)
|
||||
|
||||
|
||||
def test_the_requests_identify_the_tool(tree, module):
|
||||
_fetch(module.url("/mod/"), name="mod")
|
||||
assert module.requests and all(agent.startswith("chemenu-wikitool/") for _, agent in module.requests)
|
||||
|
||||
|
||||
def test_the_folder_is_accepted_by_raw_accept_unchanged(tree, module):
|
||||
_fetch(module.url("/mod/"), name="mod")
|
||||
snapshot = {p.name: p.read_bytes() for p in (tree / "incoming/mod").iterdir()}
|
||||
|
||||
raw_accept_command(
|
||||
files=[tree / "incoming/mod"], fidelity="published", authority="reporting",
|
||||
page=None, replaces=None, dry_run=False,
|
||||
)
|
||||
|
||||
accepted = [p for p in (tree / "raw").rglob("*") if p.is_dir() and p.name == "mod"]
|
||||
assert len(accepted) == 1
|
||||
assert {p.name: p.read_bytes() for p in accepted[0].iterdir()} == snapshot
|
||||
assert not (tree / "incoming/mod").exists()
|
||||
|
||||
|
||||
def test_the_pause_is_taken_before_each_subpage(tree, module, monkeypatch):
|
||||
write_settings(tree, monkeypatch, sites={"pause_seconds": 0.25})
|
||||
slept: list[float] = []
|
||||
monkeypatch.setattr("chemenu.commands.raw.fetch.time.sleep", slept.append)
|
||||
_fetch(module.url("/mod/"), name="mod")
|
||||
assert slept == [0.25, 0.25, 0.25]
|
||||
|
||||
|
||||
# --- refusals: nothing is written --------------------------------------------
|
||||
|
||||
|
||||
def test_robots_forbidding_a_subpage_refuses_the_fetch_and_names_it(tree, module, capsys):
|
||||
module.routes["/robots.txt"] = (200, {"Content-Type": "text/plain"}, b"User-agent: *\nDisallow: /mod/b\n")
|
||||
before = _snapshot(tree)
|
||||
_refused(module.url("/mod/"), name="mod")
|
||||
|
||||
assert _snapshot(tree) == before
|
||||
assert module.url("/mod/b?v=1") in capsys.readouterr().out.replace("\n", "")
|
||||
assert not {"/mod/a?v=1", "/mod/b?v=1", "/mod/c"} & set(_requested(module))
|
||||
|
||||
|
||||
def test_a_robots_txt_for_another_agent_does_not_apply(tree, module):
|
||||
module.routes["/robots.txt"] = (
|
||||
200, {"Content-Type": "text/plain"}, b"User-agent: somebodyelse\nDisallow: /\n",
|
||||
)
|
||||
_fetch(module.url("/mod/"), name="mod")
|
||||
assert (tree / "incoming/mod/c.html").is_file()
|
||||
|
||||
|
||||
def test_a_missing_robots_txt_allows_everything(tree, module):
|
||||
assert "/robots.txt" not in module.routes
|
||||
_fetch(module.url("/mod/"), name="mod")
|
||||
assert (tree / "incoming/mod/c.html").is_file()
|
||||
|
||||
|
||||
def test_a_robots_txt_that_cannot_be_read_refuses_the_fetch(tree, module, capsys):
|
||||
module.routes["/robots.txt"] = (503, {}, b"later")
|
||||
before = _snapshot(tree)
|
||||
_refused(module.url("/mod/"), name="mod")
|
||||
assert _snapshot(tree) == before
|
||||
assert "robots.txt" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_robots_does_not_apply_to_the_page_without_scope(tree, module):
|
||||
module.routes["/robots.txt"] = (200, {"Content-Type": "text/plain"}, b"User-agent: *\nDisallow: /\n")
|
||||
_fetch(module.url("/mod/"), name="mod", scope=False)
|
||||
assert (tree / "incoming/mod.html").is_file()
|
||||
assert "/robots.txt" not in _requested(module)
|
||||
|
||||
|
||||
def test_a_failing_third_fetch_leaves_incoming_as_it_was(tree, module):
|
||||
module.routes["/mod/b?v=1"] = (500, {}, b"broken")
|
||||
(tree / "incoming/other.md").write_text("other", encoding="utf-8")
|
||||
before = _snapshot(tree)
|
||||
_refused(module.url("/mod/"), name="mod")
|
||||
|
||||
assert _snapshot(tree) == before
|
||||
assert "/mod/b?v=1" in _requested(module)
|
||||
assert "/mod/c" not in _requested(module)
|
||||
|
||||
|
||||
def test_a_subpage_that_is_not_html_refuses_the_fetch(tree, module):
|
||||
module.routes["/mod/c"] = (200, {"Content-Type": "application/pdf"}, b"%PDF-1.7")
|
||||
before = _snapshot(tree)
|
||||
_refused(module.url("/mod/"), name="mod")
|
||||
assert _snapshot(tree) == before
|
||||
|
||||
|
||||
def test_more_subpages_than_the_maximum_refuse_before_the_first_subpage(tree, module, monkeypatch, capsys):
|
||||
write_settings(tree, monkeypatch, sites={"max_pages": 2})
|
||||
before = _snapshot(tree)
|
||||
_refused(module.url("/mod/"), name="mod")
|
||||
|
||||
assert _snapshot(tree) == before
|
||||
assert _requested(module) == ["/mod/"]
|
||||
assert "max_pages" in " ".join(capsys.readouterr().out.split())
|
||||
|
||||
|
||||
def test_a_raised_maximum_takes_effect(tree, module, monkeypatch):
|
||||
write_settings(tree, monkeypatch, sites={"max_pages": 4})
|
||||
_fetch(module.url("/mod/"), name="mod")
|
||||
assert (tree / "incoming/mod/c.md").is_file()
|
||||
|
||||
|
||||
def test_an_existing_folder_is_never_overwritten(tree, module):
|
||||
(tree / "incoming/mod").mkdir()
|
||||
(tree / "incoming/mod/mine.md").write_text("mine", encoding="utf-8")
|
||||
before = _snapshot(tree)
|
||||
_refused(module.url("/mod/"), name="mod")
|
||||
assert _snapshot(tree) == before
|
||||
assert not {"/mod/a?v=1", "/mod/b?v=1", "/mod/c"} & set(_requested(module))
|
||||
|
||||
|
||||
def test_a_path_over_the_budget_is_refused_before_a_subpage_is_requested(tree, module):
|
||||
before = _snapshot(tree)
|
||||
_refused(module.url("/mod/"), name="m" * 140)
|
||||
assert _snapshot(tree) == before
|
||||
assert "/mod/a?v=1" not in _requested(module)
|
||||
|
||||
|
||||
def test_a_page_whose_handler_offers_no_subpages_is_refused(tree, module):
|
||||
module.html("/plain", _sub("plain"))
|
||||
before = _snapshot(tree)
|
||||
_refused(module.url("/plain"), name="plain")
|
||||
assert _snapshot(tree) == before
|
||||
|
||||
|
||||
def test_an_index_without_subpages_is_refused(tree, module):
|
||||
module.html("/mod/", b"<html><body><main><p>No table at all, only text.</p></main></body></html>")
|
||||
before = _snapshot(tree)
|
||||
_refused(module.url("/mod/"), name="mod")
|
||||
assert _snapshot(tree) == before
|
||||
|
||||
|
||||
def test_scope_does_not_go_with_html(tree):
|
||||
saved = tree / "incoming/mod.html"
|
||||
saved.write_bytes(b"<p>x</p>")
|
||||
with pytest.raises(typer.Exit) as excinfo:
|
||||
raw_fetch_command(url=None, html=saved, source_url="http://127.0.0.1/mod/", name=None, scope=True)
|
||||
assert excinfo.value.exit_code == 1
|
||||
|
||||
|
||||
# --- the building block as data ---------------------------------------------
|
||||
|
||||
|
||||
def test_the_stack_handler_for_a_module_offers_its_cmdlet_pages_as_subpages():
|
||||
special = next(s for s in sites.all_sites() if s.name == "learn-microsoft-powershell-module")
|
||||
assert special.subpages is not None and special.subpages.prefix == "{page}"
|
||||
html = (
|
||||
'<table><tr><td><a data-linktype="relative-path" href="get-x?view=v">x</a></td></tr>'
|
||||
'<tr><td><a data-linktype="absolute-path" href="/en-us/other">y</a></td></tr></table>'
|
||||
'<a data-linktype="relative-path" href="get-y">outside the table</a>'
|
||||
)
|
||||
assert special.subpages.urls(html, "https://learn.microsoft.com/en-us/powershell/module/m/?view=v") == [
|
||||
"https://learn.microsoft.com/en-us/powershell/module/m/get-x?view=v",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("subpages", [
|
||||
{},
|
||||
{"links": {}},
|
||||
{"links": {"tag": "a"}, "prefix": "relative/"},
|
||||
{"links": {"tag": "a"}, "extra": 1},
|
||||
])
|
||||
def test_a_malformed_subpages_block_is_a_rejected_file(subpages):
|
||||
from chemenu.sites import data
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
data.build("x", {"host": "example.org", "summary": "s", "subpages": subpages}, "instance")
|
||||
|
||||
|
||||
def test_a_literal_prefix_replaces_the_page_path():
|
||||
block = sites.Subpages(web_capture.Rule(tag="a"), "/docs/ref/")
|
||||
html = '<a href="/docs/ref/one">1</a><a href="/docs/guide/two">2</a><a href="/docs/ref/">self</a>'
|
||||
assert block.urls(html, "https://example.org/docs/index") == ["https://example.org/docs/ref/one"]
|
||||
|
||||
|
||||
def test_the_new_settings_have_their_defaults_and_refuse_nonsense():
|
||||
assert check(SITES, {}).values["max_pages"] == 200
|
||||
assert check(SITES, {}).values["pause_seconds"] == 1
|
||||
assert check(SITES, {"max_pages": 0}).problems
|
||||
assert check(SITES, {"pause_seconds": -1}).problems
|
||||
assert not check(SITES, {"pause_seconds": 0}).problems
|
||||
@@ -60,7 +60,7 @@ def _handler(root: Path, name: str, content) -> Path:
|
||||
|
||||
|
||||
def _fetch(url, name=None, html=None):
|
||||
return raw_fetch_command(url=None if html else url, html=html, source_url=url if html else None, name=name)
|
||||
return raw_fetch_command(url=None if html else url, html=html, source_url=url if html else None, name=name, scope=False)
|
||||
|
||||
|
||||
def _refused(url, **kwargs) -> None:
|
||||
|
||||
@@ -57,8 +57,8 @@ def served(monkeypatch):
|
||||
return asked
|
||||
|
||||
|
||||
def _fetch(url=None, html=None, source_url=None, name=None):
|
||||
return raw_fetch_command(url=url, html=html, source_url=source_url, name=name)
|
||||
def _fetch(url=None, html=None, source_url=None, name=None, scope=False):
|
||||
return raw_fetch_command(url=url, html=html, source_url=source_url, name=name, scope=scope)
|
||||
|
||||
|
||||
def _refused(**kwargs) -> None:
|
||||
@@ -130,13 +130,13 @@ def test_the_stem_comes_from_the_standard_form(tree, served):
|
||||
assert (tree / "incoming/get-oslicenseinfo.md").is_file()
|
||||
|
||||
|
||||
def test_the_checks_name_the_source_and_the_cmdlet_pages(tree, served, capsys):
|
||||
def test_the_checks_name_the_source_and_the_subpages(tree, served, capsys):
|
||||
_fetch(MODULE_URL, name="oslicense")
|
||||
lines = _checks(capsys)
|
||||
|
||||
assert len(lines) == 2
|
||||
assert SOURCE_URL in lines[0]
|
||||
assert "11 cmdlet page(s)" in lines[1]
|
||||
assert "index with 11 subpage(s)" in lines[1] and "--scope" in lines[1]
|
||||
|
||||
|
||||
def test_a_page_of_the_general_handler_has_only_the_source_check(tree, served, capsys):
|
||||
@@ -188,7 +188,7 @@ def test_html_applies_the_rules_but_keeps_the_url_as_typed(tree, monkeypatch, ca
|
||||
assert "requested_url" not in header
|
||||
assert header["site"] == "learn-microsoft-powershell-module"
|
||||
assert "Exit editor mode" not in text and "# OSLicense Module" in text
|
||||
assert len(_checks(capsys)) == 2
|
||||
assert len(_checks(capsys)) == 1
|
||||
|
||||
|
||||
# --- the instance's part -----------------------------------------------------
|
||||
@@ -286,7 +286,8 @@ def test_a_special_handler_inherits_from_the_general_one():
|
||||
assert special.drop == general.drop
|
||||
assert special.rewrites == general.rewrites
|
||||
assert special.checks[: len(general.checks)] == general.checks
|
||||
assert len(special.checks) == len(general.checks) + 1
|
||||
assert special.checks == general.checks
|
||||
assert special.subpages is not None and general.subpages is None
|
||||
|
||||
|
||||
def test_a_duplicate_host_and_path_is_a_registration_error(registry):
|
||||
|
||||
Reference in new issue
Block a user