feat: raw fetch - a sanctioned intake for a URL into incoming/, HTML as received plus derived text (#120)

Files changed:
- CHANGES.md
- README.md
- VERSION
- instructions/wiki-ingest/SKILL.md
- raw/CONTRACT.md
- tools/CONTRACT.md
- tools/README.md
- tools/chemenu/cli_contract.py
- tools/chemenu/commands/raw_cmd.py
- tools/chemenu/tests/test_cli.py
- tools/chemenu/tests/test_portability.py
- tools/chemenu/tests/test_raw_fetch.py
- tools/chemenu/web_capture.py

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2

tools/chemenu/tests/test_cli.py

@@ -338,10 +338,13 @@ def test_network_yes_is_exactly_the_commands_that_can_reach_outside_this_checkou
     """Gitea #144: `network:` means *any* reach outside this checkout - an HTTP call
     `wikitool` makes itself, or a git operation against a remote (fetch/ls-remote/push) -
     not only the two `version_cmd.py` used to claim exclusivity for. `dist upgrade` is on the list
-    since `--latest`, which asks the release feed and downloads from it. Pinned as an explicit
+    since `--latest`, which asks the release feed and downloads from it, and `raw fetch`
+    since it exists (Gitea #120) - its `--html` form stays offline, which does not turn the
+    command back to `no`. Pinned as an explicit
     set so a command gaining or losing that reach is a deliberate edit here, not a silent
     drift between the property and what the command actually does."""
     expected = {
+        "raw fetch",
         "sync",
         "publish",
         "version check",

tools/chemenu/tests/test_portability.py

@@ -26,9 +26,10 @@ from chemenu import filelock
 TOOLS_DIR = Path(__file__).resolve().parents[2]
 PACKAGE_DIR = TOOLS_DIR / "chemenu"
 
-# Receivers whose `open` is not a text-file open: `os.open` takes flags, and the
-# archive modules open binary members.
-NOT_TEXT_OPEN = {"os", "tarfile", "zipfile", "gzip"}
+# Receivers whose `open` is not a text-file open: `os.open` takes flags, the
+# archive modules open binary members, and `opener` is a `urllib`
+# `OpenerDirector` (`web_capture.fetch`), whose `open` sends an HTTP request.
+NOT_TEXT_OPEN = {"os", "tarfile", "zipfile", "gzip", "opener"}
 
 # The one module allowed to import the platform lock modules.
 LOCK_MODULE = PACKAGE_DIR / "filelock.py"

tools/chemenu/tests/test_raw_fetch.py

@@ -0,0 +1,429 @@
+"""`wikitool raw fetch` - the sanctioned intake for a URL (Gitea #120).
+
+Every fetch goes to a real `http.server` on 127.0.0.1, never to the network:
+what is under test is what arrives on the wire and what lands on disk - the
+exact bytes, the charset a header did or did not declare, a redirect, a body
+that is too large or too slow - and a fetcher stub would have to fake exactly
+the parts that matter.
+"""
+from __future__ import annotations
+
+import datetime
+import http.server
+import threading
+import time
+import urllib.request
+from pathlib import Path
+
+import pytest
+import typer
+import yaml
+
+from chemenu import web_capture
+from chemenu.commands.raw_cmd import raw_accept_command, raw_fetch_command
+
+ARTICLE = (
+    b"<!doctype html><html><head><meta charset=\"utf-8\"><title>A Post</title>"
+    b"<script>var tracking = 'SCRIPT-TEXT';</script><style>.x{}</style></head><body>"
+    b"<nav><a href=\"/\">NAV-TEXT</a></nav>"
+    b"<main><h1>Main heading</h1><p>First paragraph with a <a href=\"other\">relative link</a>.</p>"
+    b"<ul><li>one</li><li>two</li></ul><pre>code  line</pre></main>"
+    b"<aside>ASIDE-TEXT</aside><footer>FOOTER-TEXT</footer></body></html>"
+)
+
+
+class Site:
+    """Serves `routes[path] = (status, headers, body)`; a body that is a
+    callable gets the handler instead, for the slow and the unbounded cases.
+    Records every request's path and User-Agent."""
+
+    def __init__(self) -> None:
+        self.routes: dict = {}
+        self.requests: list[tuple[str, str]] = []
+        outer = self
+
+        class Handler(http.server.BaseHTTPRequestHandler):
+            def do_GET(self):  # noqa: N802 - http.server's name
+                outer.requests.append((self.path, self.headers.get("User-Agent", "")))
+                status, headers, body = outer.routes.get(self.path, (404, {}, b"not found"))
+                self.send_response(status)
+                for key, value in headers.items():
+                    self.send_header(key, value)
+                if callable(body):
+                    self.end_headers()
+                    body(self)
+                    return
+                if "Content-Length" not in headers:
+                    self.send_header("Content-Length", str(len(body)))
+                self.end_headers()
+                self.wfile.write(body)
+
+            def log_message(self, *args):  # silence the test output
+                pass
+
+        self.httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+        self.httpd.daemon_threads = True
+        self.thread = threading.Thread(
+            target=self.httpd.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True
+        )
+        self.thread.start()
+
+    def url(self, path: str) -> str:
+        return f"http://127.0.0.1:{self.httpd.server_address[1]}{path}"
+
+    def html(self, path: str, body: bytes, content_type: str = "text/html; charset=utf-8") -> str:
+        self.routes[path] = (200, {"Content-Type": content_type}, body)
+        return self.url(path)
+
+    def stop(self) -> None:
+        self.httpd.shutdown()
+        self.httpd.server_close()
+
+
+@pytest.fixture
+def site(monkeypatch: pytest.MonkeyPatch):
+    # A developer's or CI runner's proxy must not see a loopback request.
+    for var in ("http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"):
+        monkeypatch.delenv(var, raising=False)
+    server = Site()
+    yield server
+    server.stop()
+
+
+@pytest.fixture
+def tree(kb_dir):
+    """kb_dir repoints config.ROOT at tmp_path; add raw/ and incoming/."""
+    root = kb_dir.parent
+    (root / "raw").mkdir()
+    (root / "incoming").mkdir(exist_ok=True)
+    return root
+
+
+def _fetch(url=None, html=None, source_url=None, name=None):
+    return raw_fetch_command(url=url, html=html, source_url=source_url, name=name)
+
+
+def _refused(**kwargs) -> None:
+    with pytest.raises(typer.Exit) as excinfo:
+        _fetch(**kwargs)
+    assert excinfo.value.exit_code == 1
+
+
+def _snapshot(root: Path) -> dict[str, bytes]:
+    return {str(p.relative_to(root)): p.read_bytes() for p in sorted(root.rglob("*")) if p.is_file()}
+
+
+def _header_and_text(md: Path) -> tuple[dict, str]:
+    content = md.read_text(encoding="utf-8")
+    assert content.startswith("---\n")
+    head, _, text = content[4:].partition("\n---\n")
+    return yaml.safe_load(head), text
+
+
+# --- the fetched bundle -----------------------------------------------------
+
+
+def test_fetch_writes_html_and_md_to_incoming_and_leaves_raw_untouched(tree, site):
+    url = site.html("/blog/post.html", ARTICLE)
+    raw_before = _snapshot(tree / "raw")
+
+    _fetch(url)
+
+    assert sorted(p.name for p in (tree / "incoming").iterdir()) == ["post.html", "post.md"]
+    assert _snapshot(tree / "raw") == raw_before
+
+
+def test_the_html_is_byte_identical_to_what_the_server_sent(tree, site):
+    body = ARTICLE + b"\r\n<!-- trailing CRLF and \xc3\xa4 bytes -->\r\n"
+    _fetch(site.html("/post", body))
+    assert (tree / "incoming/post.html").read_bytes() == body
+
+
+def test_header_records_the_capture(tree, site):
+    url = site.html("/post", ARTICLE)
+    _fetch(url)
+    header, _ = _header_and_text(tree / "incoming/post.md")
+
+    assert header["fetched_by"] == "wikitool raw fetch"
+    assert header["url"] == url
+    assert header["final_url"] == url
+    assert header["http_status"] == 200
+    assert header["content_type"] == "text/html; charset=utf-8"
+    assert header["charset"] == "utf-8 (from: header)"
+    assert header["title"] == "A Post"
+    assert header["derived_from"] == "post.html"
+    # YAML reads the ISO-8601 UTC stamp back as an aware datetime.
+    retrieved = header["retrieved"]
+    assert retrieved.tzinfo is not None and retrieved.utcoffset() == datetime.timedelta(0)
+    assert abs(datetime.datetime.now(datetime.timezone.utc) - retrieved) < datetime.timedelta(minutes=5)
+    assert "author" not in header
+
+
+def test_the_request_identifies_the_tool(tree, site):
+    _fetch(site.html("/post", ARTICLE))
+    assert site.requests[0][1].startswith("chemenu-wikitool/")
+
+
+def test_two_fetches_of_the_same_bytes_derive_identical_text(tree, site):
+    url = site.html("/post", ARTICLE)
+    _fetch(url, name="first")
+    _fetch(url, name="second")
+
+    def without_retrieved(path: Path) -> list[str]:
+        lines = path.read_text(encoding="utf-8").splitlines()
+        return [line.replace("first.html", "X").replace("second.html", "X")
+                for line in lines if not line.startswith("retrieved:")]
+
+    assert without_retrieved(tree / "incoming/first.md") == without_retrieved(tree / "incoming/second.md")
+
+
+def test_boilerplate_is_dropped_and_main_is_kept_whole(tree, site):
+    _fetch(site.html("/post", ARTICLE))
+    _, text = _header_and_text(tree / "incoming/post.md")
+
+    for dropped in ("NAV-TEXT", "FOOTER-TEXT", "ASIDE-TEXT", "SCRIPT-TEXT", ".x{}"):
+        assert dropped not in text
+    assert "# Main heading" in text
+    assert "First paragraph with a [relative link](" in text
+    assert "- one\n- two" in text
+    assert "```\ncode  line\n```" in text
+
+
+def test_relative_links_resolve_against_the_final_url_after_a_redirect(tree, site):
+    target = site.html("/articles/post", ARTICLE)
+    site.routes["/old"] = (301, {"Location": "/articles/post"}, b"")
+    _fetch(site.url("/old"), name="post")
+
+    header, text = _header_and_text(tree / "incoming/post.md")
+    assert header["url"] == site.url("/old")
+    assert header["final_url"] == target
+    assert f"[relative link]({site.url('/articles/other')})" in text
+
+
+def test_latin1_declared_only_in_meta_is_decoded(tree, site):
+    body = (
+        "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-8859-1\">"
+        "<title>Umlaute</title></head><body><main><p>Größe, Übermaß, Äpfel</p></main></body></html>"
+    ).encode("iso-8859-1")
+    _fetch(site.html("/umlaute", body, content_type="text/html"))
+
+    header, text = _header_and_text(tree / "incoming/umlaute.md")
+    assert header["charset"] == "iso-8859-1 (from: meta)"
+    assert "Größe, Übermaß, Äpfel" in text
+    assert (tree / "incoming/umlaute.html").read_bytes() == body
+
+
+def test_the_header_charset_wins_over_meta_and_the_bom_over_both():
+    meta_says_latin1 = b'<meta charset="iso-8859-1"><p>\xc3\xa4</p>'
+    assert web_capture.decode(meta_says_latin1, "text/html; charset=utf-8").text.endswith("<p>ä</p>")
+    decoded = web_capture.decode(b"\xef\xbb\xbf" + meta_says_latin1, "text/html; charset=iso-8859-1")
+    assert (decoded.charset, decoded.source) == ("utf-8", "bom")
+    assert web_capture.decode(b"<p>x</p>", None).source == "default"
+
+
+def test_undecodable_bytes_are_replaced_and_reported(tree, site, capsys):
+    body = b"<html><body><main><p>broken \xff\xfe here and enough text " + b"x" * 300 + b"</p></main></body></html>"
+    _fetch(site.html("/broken", body))
+    _, text = _header_and_text(tree / "incoming/broken.md")
+    assert "broken �� here" in text
+    assert "undecodable byte" in capsys.readouterr().out
+
+
+def test_a_nearly_empty_page_is_written_with_a_warning(tree, site, capsys):
+    body = b"<html><body><div id=\"app\"></div><script>render()</script></body></html>"
+    _fetch(site.html("/spa", body))
+    assert (tree / "incoming/spa.md").is_file()
+    assert "only 0 characters" in capsys.readouterr().out
+
+
+def test_the_bundle_is_accepted_into_raw_as_one_source(tree, site):
+    _fetch(site.html("/blog/post", ARTICLE))
+    raw_accept_command(
+        files=[tree / "incoming/post.html", tree / "incoming/post.md"],
+        fidelity="published", authority="reporting", page=None, replaces=None, dry_run=False,
+    )
+    today = datetime.date.today()
+    bundle = tree / "raw" / f"{today.year:04d}" / f"{today.month:02d}" / "post"
+    assert sorted(p.name for p in bundle.iterdir()) == ["post.html", "post.md"]
+    assert (bundle / "post.html").read_bytes() == ARTICLE
+
+
+# --- refusals ---------------------------------------------------------------
+
+
+@pytest.mark.parametrize("existing", ["post.html", "post.md"])
+def test_an_existing_target_is_never_overwritten(tree, site, existing):
+    held = tree / "incoming" / existing
+    held.write_bytes(b"already here")
+    _refused(url=site.html("/post", ARTICLE))
+    assert held.read_bytes() == b"already here"
+    assert [p.name for p in (tree / "incoming").iterdir()] == [existing]
+
+
+def test_a_file_url_is_refused_without_writing(tree):
+    _refused(url="file:///etc/passwd")
+    assert list((tree / "incoming").iterdir()) == []
+
+
+def test_a_redirect_to_another_scheme_is_refused(tree, site):
+    site.routes["/sneaky"] = (302, {"Location": "ftp://127.0.0.1/secret"}, b"")
+    _refused(url=site.url("/sneaky"))
+    assert list((tree / "incoming").iterdir()) == []
+
+
+@pytest.mark.parametrize("declared", [True, False])
+def test_a_response_over_the_size_limit_is_refused(tree, site, monkeypatch, declared):
+    monkeypatch.setattr(web_capture, "MAX_BYTES", 1024)
+    body = b"<p>" + b"x" * 4096 + b"</p>"
+    if declared:
+        site.html("/big", body)
+    else:
+        def stream(handler):  # no Content-Length: HTTP/1.0, the body ends at close
+            handler.wfile.write(body)
+        site.routes["/big"] = (200, {"Content-Type": "text/html"}, stream)
+    _refused(url=site.url("/big"))
+    assert list((tree / "incoming").iterdir()) == []
+
+
+def test_a_slow_server_times_out(tree, site, monkeypatch):
+    monkeypatch.setattr(web_capture, "TIMEOUT_SECONDS", 0.3)
+
+    def stall(handler):
+        handler.wfile.write(b"<p>start")
+        handler.wfile.flush()
+        time.sleep(1.5)
+
+    site.routes["/slow"] = (200, {"Content-Type": "text/html"}, stall)
+    _refused(url=site.url("/slow"))
+    assert list((tree / "incoming").iterdir()) == []
+
+
+def test_an_http_error_is_refused(tree, site):
+    _refused(url=site.url("/missing"))
+    assert list((tree / "incoming").iterdir()) == []
+
+
+def test_url_and_html_are_exclusive(tree):
+    page = tree / "incoming/post.html"
+    page.write_bytes(ARTICLE)
+    _refused(url="https://example.org/post", html=page, source_url="https://example.org/post")
+    _refused()
+    _refused(url="https://example.org/post", source_url="https://example.org/post")
+
+
+# --- non-HTML responses -----------------------------------------------------
+
+
+def test_a_pdf_is_stored_byte_identical_without_a_derivation(tree, site):
+    body = b"%PDF-1.7\n\x00\x01binary\xff"
+    site.routes["/paper.pdf"] = (200, {"Content-Type": "application/pdf"}, body)
+    _fetch(site.url("/paper.pdf"))
+    assert [p.name for p in (tree / "incoming").iterdir()] == ["paper.pdf"]
+    assert (tree / "incoming/paper.pdf").read_bytes() == body
+
+
+def test_plain_text_is_stored_as_txt_unchanged(tree, site):
+    site.routes["/notes"] = (200, {"Content-Type": "text/plain; charset=utf-8"}, b"line one\r\nline two\n")
+    _fetch(site.url("/notes"))
+    assert [p.name for p in (tree / "incoming").iterdir()] == ["notes.txt"]
+    assert (tree / "incoming/notes.txt").read_bytes() == b"line one\r\nline two\n"
+
+
+# --- --html: a page saved from a browser ------------------------------------
+
+
+def test_html_derives_only_the_md_without_network_and_matches_a_fetch(tree, site, monkeypatch):
+    url = site.html("/blog/post", ARTICLE)
+    _fetch(url, name="online")
+
+    saved = tree / "incoming/post.html"
+    saved.write_bytes(ARTICLE)
+
+    def no_network(*args, **kwargs):
+        raise AssertionError("--html must not touch the network")
+
+    monkeypatch.setattr(web_capture, "fetch", no_network)
+    monkeypatch.setattr(urllib.request, "urlopen", no_network)
+    monkeypatch.setattr(urllib.request.OpenerDirector, "open", no_network)
+    _fetch(html=saved, source_url=url)
+
+    assert saved.read_bytes() == ARTICLE
+    assert sorted(p.name for p in (tree / "incoming").iterdir()) == [
+        "online.html", "online.md", "post.html", "post.md",
+    ]
+    header, text = _header_and_text(tree / "incoming/post.md")
+    _, online_text = _header_and_text(tree / "incoming/online.md")
+    assert text == online_text
+    assert header["fetched_by"] == "wikitool raw fetch --html"
+    assert header["url"] == url
+    assert header["derived_from"] == "post.html"
+    assert "derived" in header
+    for absent in ("retrieved", "final_url", "http_status", "content_type"):
+        assert absent not in header
+    assert header["charset"] == "utf-8 (from: meta)"
+
+
+def test_html_outside_incoming_is_refused(tree):
+    outside = tree / "saved.html"
+    outside.write_bytes(ARTICLE)
+    _refused(html=outside, source_url="https://example.org/post")
+    assert not (tree / "saved.md").exists()
+    assert list((tree / "incoming").iterdir()) == []
+
+
+def test_html_without_url_is_refused(tree):
+    saved = tree / "incoming/post.html"
+    saved.write_bytes(ARTICLE)
+    _refused(html=saved)
+    assert [p.name for p in (tree / "incoming").iterdir()] == ["post.html"]
+
+
+def test_html_does_not_overwrite_an_existing_md(tree):
+    saved = tree / "incoming/post.html"
+    saved.write_bytes(ARTICLE)
+    (tree / "incoming/post.md").write_text("mine", encoding="utf-8")
+    _refused(html=saved, source_url="https://example.org/post")
+    assert (tree / "incoming/post.md").read_text(encoding="utf-8") == "mine"
+
+
+# --- naming and the derivation itself ---------------------------------------
+
+
+@pytest.mark.parametrize(
+    "url, stem",
+    [
+        ("https://example.org/blog/My%20Post.html", "my-post"),
+        ("https://example.org/blog/über-uns/", "uber-uns"),
+        ("https://example.org/", "example-org"),
+        ("https://example.org/" + "a" * 100, "a" * 60),
+    ],
+)
+def test_the_stem_comes_from_the_url(url, stem):
+    assert web_capture.stem_for(url) == stem
+
+
+def test_name_overrides_the_stem(tree, site):
+    _fetch(site.html("/post", ARTICLE), name="chosen")
+    assert sorted(p.name for p in (tree / "incoming").iterdir()) == ["chosen.html", "chosen.md"]
+
+
+def test_name_with_a_path_separator_is_refused(tree, site):
+    _refused(url=site.html("/post", ARTICLE), name="../escape")
+    assert list((tree / "incoming").iterdir()) == []
+
+
+def test_a_single_article_is_the_root_when_there_is_no_main():
+    html = "<body><p>OUTSIDE</p><article><p>INSIDE</p></article></body>"
+    assert web_capture.derive_text(html, "https://example.org/") == "INSIDE"
+
+
+def test_two_articles_fall_back_to_body():
+    html = "<body><article><p>ONE</p></article><article><p>TWO</p></article></body>"
+    assert web_capture.derive_text(html, "https://example.org/") == "ONE\n\nTWO"
+
+
+def test_a_title_with_a_colon_keeps_the_header_parseable(tree, site):
+    body = ARTICLE.replace(b"<title>A Post</title>", b"<title>Part 2: the #1 reason</title>")
+    _fetch(site.html("/post", body))
+    header, _ = _header_and_text(tree / "incoming/post.md")
+    assert header["title"] == "Part 2: the #1 reason"