sites: raw fetch gets site handlers - standard URL form, per-website cleanup and CHECK hints as a code registry, learn.microsoft.com first, plus a sites settings section (#200); 10.0.0-beta.3
Files changed: - tools/chemenu/sites/ (registry, learn-microsoft handlers), web_capture.py, commands/raw/fetch.py, config_shapes.py - tests, fixture, raw/CONTRACT.md, tools/README.md, docs, CHANGES.md, VERSION Files changed: - CHANGES.md - VERSION - docs/how-to/ingest-a-source.md - docs/reference/commands/README.md - docs/reference/commands/raw-material-and-uploads.md - docs/reference/configuration.md - raw/CONTRACT.md - tools/CONTRACT.md - tools/README.md - tools/chemenu/commands/raw/fetch.py - tools/chemenu/config_shapes.py - tools/chemenu/reference_doc.py - tools/chemenu/sites/__init__.py - tools/chemenu/sites/learn_microsoft.py - tools/chemenu/tests/fixtures/learn_microsoft/oslicense-module.html - tools/chemenu/tests/test_config_shapes.py - tools/chemenu/tests/test_sites.py - tools/chemenu/web_capture.py Co-Authored-By: Claude Code <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
This commit is contained in:
torbenandClaude Code committed
1 parent
298b514119
commit
2213d68900
18 files changed
+1226
-27
No files matched your search
@@ -0,0 +1,194 @@
|
||||
<!DOCTYPE html>
|
||||
<html class="layout layout-holy-grail show-table-of-contents reference" lang="en-us" dir="ltr" data-authenticated="false" data-auth-status-determined="false" data-target="docs">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>OSLicense Module | Fixture Learn</title>
|
||||
<meta name="locale" content="en-us" />
|
||||
<meta name="original_content_git_url" content="https://example.invalid/docs-source/blob/live/docset/winserver2025-ps/OSLicense/OSLicense.md" />
|
||||
<script>window.appInsights = "SCRIPT-TEXT";</script>
|
||||
</head>
|
||||
<body class="luna-theme">
|
||||
<div class="header-holder has-default-focus"><header class="header">HEADER-TEXT</header></div>
|
||||
<div class="mainContainer uhfMainContainer">
|
||||
<div class="columns has-large-gaps">
|
||||
<div id="main-column" class="column is-full is-8-desktop">
|
||||
<main id="main" role="main" class="content">
|
||||
<div id="ms--content-header" class="content-header default-focus border-bottom-none">
|
||||
<button class="button button-sm contents-button" type="button" data-bi-name="contents-expand"><span>Table of contents</span></button>
|
||||
<button class="button button-sm ap-collapse-behavior" type="button" data-bi-name="editor-mode-collapse"><span>Exit editor mode</span></button>
|
||||
</div>
|
||||
<div id="action-panel" role="region" aria-label="Action Panel" class="action-panel"></div>
|
||||
<div id="article-header" class="background-color-body-medium">
|
||||
<details id="article-header-breadcrumbs-overflow-popover"><summary>Breadcrumb overflow</summary></details>
|
||||
<bread-crumbs id="article-header-breadcrumbs"><ol><li><a href="/en-us/">BREADCRUMB-TEXT</a></li></ol></bread-crumbs>
|
||||
<div id="article-header-page-actions">
|
||||
<button type="button" data-bi-name="ask-learn-assistant-entry"><span>Ask Learn</span></button>
|
||||
<button type="button" data-bi-name="ask-learn-assistant-entry"><span>Ask Learn</span></button>
|
||||
<button type="button" data-bi-name="ask-learn-assistant-entry"><span>Ask Learn</span></button>
|
||||
<details id="article-header-page-actions-overflow">
|
||||
<summary>More actions</summary>
|
||||
<div class="popover">
|
||||
<button type="button" id="ms--focus-mode-button"><span>Reading mode</span></button>
|
||||
<button type="button" class="contents-button"><span>Table of contents</span></button>
|
||||
<a id="lang-link-overflow" href="https://example.invalid/en-us/">Read in English</a>
|
||||
<button type="button" class="collection" data-list-type="collection"><span>Add</span></button>
|
||||
<button type="button" data-list-type="plan"><span>Add to Plans</span></button>
|
||||
<a href="https://example.invalid/docs-source/blob/main/docset/winserver2025-ps/OSLicense/OSLicense.md" data-contenteditbtn>Edit</a>
|
||||
<hr>
|
||||
<button type="button" data-copy-markdown><span>Copy Markdown</span></button>
|
||||
<button type="button" data-print-page><span>Print</span></button>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
</div>
|
||||
<!-- privateUnauthorizedTemplate is hidden by default -->
|
||||
<div unauthorized-private-section data-bi-name="permission-content-unauthorized-private" hidden>
|
||||
<hr class="hr margin-top-xs margin-bottom-sm" />
|
||||
<div class="notification notification-info">
|
||||
<div class="notification-content">
|
||||
<p class="margin-top-none notification-title"><span>Note</span></p>
|
||||
<p class="margin-top-none authentication-determined not-authenticated">
|
||||
Access to this page requires authorization. You can try <a class="docs-sign-in" href="#">signing in</a> or <a class="docs-change-directory">changing directories</a>.
|
||||
</p>
|
||||
<p class="margin-top-none authentication-determined authenticated">
|
||||
Access to this page requires authorization. You can try <a class="docs-change-directory">changing directories</a>.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="content"></div>
|
||||
<div class="content"><h1 class="margin-bottom-xs" data-no-chunk="">OSLicense Module</h1>
|
||||
<div class="margin-block-xxs"><ul class="metadata page-metadata" data-bi-name="page info"></ul></div>
|
||||
<nav id="center-doc-outline" class="doc-outline">OUTLINE-TEXT</nav>
|
||||
<div data-moniker-range-group><div data-moniker="windowsserver2025-ps">
|
||||
<p>The <code>OSLicense</code> module provides commands to read and configure the activation of an operating system.</p>
|
||||
<h2 id="oslicense">OSLicense</h2>
|
||||
<table class="table" data-no-chunk="">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Cmdlet</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<a href="get-adlicenseinfo?view=windowsserver2025-ps" data-linktype="relative-path">Get-ADLicenseInfo</a>
|
||||
</td>
|
||||
<td>
|
||||
<p>Reads Active Directory-based activation information.</p>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<a href="get-kmslicenseinfo?view=windowsserver2025-ps" data-linktype="relative-path">Get-KmsLicenseInfo</a>
|
||||
</td>
|
||||
<td>
|
||||
<p>Reads Key Management Service information.</p>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<a href="get-oslicenseinfo?view=windowsserver2025-ps" data-linktype="relative-path">Get-OSLicenseInfo</a>
|
||||
</td>
|
||||
<td>
|
||||
<p>Reads operating system licensing information.</p>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<a href="get-subscriptionlicenseinfo?view=windowsserver2025-ps" data-linktype="relative-path">Get-SubscriptionLicenseInfo</a>
|
||||
</td>
|
||||
<td>
|
||||
<p>Reads subscription activation information.</p>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<a href="invoke-adlicense?view=windowsserver2025-ps" data-linktype="relative-path">Invoke-ADLicense</a>
|
||||
</td>
|
||||
<td>
|
||||
<p>Runs Active Directory-based activation.</p>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<a href="invoke-kmslicense?view=windowsserver2025-ps" data-linktype="relative-path">Invoke-KmsLicense</a>
|
||||
</td>
|
||||
<td>
|
||||
<p>Runs Key Management Service activation.</p>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<a href="invoke-oslicense?view=windowsserver2025-ps" data-linktype="relative-path">Invoke-OSLicense</a>
|
||||
</td>
|
||||
<td>
|
||||
<p>Runs operating system activation.</p>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<a href="invoke-subscriptionlicense?view=windowsserver2025-ps" data-linktype="relative-path">Invoke-SubscriptionLicense</a>
|
||||
</td>
|
||||
<td>
|
||||
<p>Runs subscription activation.</p>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<a href="set-kmslicenseinfo?view=windowsserver2025-ps" data-linktype="relative-path">Set-KmsLicenseInfo</a>
|
||||
</td>
|
||||
<td>
|
||||
<p>Configures Key Management Service activation.</p>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<a href="set-oslicenseinfo?view=windowsserver2025-ps" data-linktype="relative-path">Set-OSLicenseInfo</a>
|
||||
</td>
|
||||
<td>
|
||||
<p>Configures operating system activation.</p>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<a href="set-subscriptionlicenseinfo?view=windowsserver2025-ps" data-linktype="relative-path">Set-SubscriptionLicenseInfo</a>
|
||||
</td>
|
||||
<td>
|
||||
<p>Configures subscription activation.</p>
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div></div>
|
||||
</div>
|
||||
<section id="site-user-feedback-footer" class="font-size-sm margin-top-md">
|
||||
<h2 id="feedback" class="margin-top-sm-tablet">Feedback</h2>
|
||||
<p>Was this page helpful?</p>
|
||||
<button type="button"><span>Yes</span></button>
|
||||
<button type="button"><span>No</span></button>
|
||||
<details id="mobile-help-popover"><summary>Need help with this topic?</summary><p>Want to try using Ask Learn to clarify or guide you through this topic? Suggest a fix?</p></details>
|
||||
</section>
|
||||
<div id="ms--inline-notifications" class="margin-block-xs"></div>
|
||||
<div id="assertive-live-region" role="alert" aria-live="assertive" class="visually-hidden">LIVE-REGION-TEXT</div>
|
||||
<div id="polite-live-region" role="status" aria-live="polite" class="visually-hidden">LIVE-REGION-TEXT</div>
|
||||
</main>
|
||||
</div>
|
||||
<aside id="layout-body-aside" class="layout-body-aside">RIGHT-RAIL-TEXT</aside>
|
||||
</div>
|
||||
</div>
|
||||
<footer id="footer">FOOTER-TEXT</footer>
|
||||
</body>
|
||||
</html>
|
||||
@@ -78,7 +78,7 @@ def test_the_telemetry_policy_reads_the_record():
|
||||
|
||||
def test_one_record_per_section():
|
||||
assert [shape.key for shape in config_shapes.SETTINGS_SECTIONS] == [
|
||||
"telemetry", "remotes", "upload", "tasks", "superproductivity", "caldav",
|
||||
"telemetry", "remotes", "upload", "tasks", "superproductivity", "caldav", "sites",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
"""The site handlers behind `raw fetch` (`chemenu.sites`).
|
||||
|
||||
The page under `fixtures/learn_microsoft/` is a rebuild of the structure of a
|
||||
learn.microsoft.com module overview page, written for this suite. The handlers
|
||||
are keyed by host name, so a fetch of that host is stood in for by replacing
|
||||
`web_capture.fetch` - the network itself is covered by `test_raw_fetch.py`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import typer
|
||||
import yaml
|
||||
|
||||
from chemenu import sites, web_capture
|
||||
from chemenu.commands.raw.fetch import raw_fetch_command
|
||||
from chemenu.tests.conftest import write_settings
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures/learn_microsoft/oslicense-module.html"
|
||||
PAGE = FIXTURE.read_bytes()
|
||||
MODULE_URL = "https://learn.microsoft.com/de-de/powershell/module/oslicense/?view=windowsserver2025-ps"
|
||||
STANDARD_URL = "https://learn.microsoft.com/en-us/powershell/module/oslicense/?view=windowsserver2025-ps"
|
||||
CMDLETS = [
|
||||
"Get-ADLicenseInfo", "Get-KmsLicenseInfo", "Get-OSLicenseInfo", "Get-SubscriptionLicenseInfo",
|
||||
"Invoke-ADLicense", "Invoke-KmsLicense", "Invoke-OSLicense", "Invoke-SubscriptionLicense",
|
||||
"Set-KmsLicenseInfo", "Set-OSLicenseInfo", "Set-SubscriptionLicenseInfo",
|
||||
]
|
||||
CHROME = [
|
||||
"requires authorization", "Exit editor mode", "Add to Plans", "Reading mode", "Copy Markdown",
|
||||
"Ask Learn", "BREADCRUMB-TEXT", "OUTLINE-TEXT", "HEADER-TEXT", "FOOTER-TEXT", "RIGHT-RAIL-TEXT",
|
||||
"LIVE-REGION-TEXT", "Was this page helpful", "SCRIPT-TEXT",
|
||||
]
|
||||
SOURCE_URL = "https://example.invalid/docs-source/blob/live/docset/winserver2025-ps/OSLicense/OSLicense.md"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tree(kb_dir):
|
||||
root = kb_dir.parent
|
||||
(root / "raw").mkdir()
|
||||
(root / "incoming").mkdir(exist_ok=True)
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def served(monkeypatch):
|
||||
"""Every fetch answers with the fixture page; the URLs asked for are recorded."""
|
||||
asked: list[str] = []
|
||||
|
||||
def fake(url, user_agent, timeout=0, max_bytes=0):
|
||||
asked.append(url)
|
||||
return web_capture.Response(url, url, 200, "text/html; charset=utf-8", PAGE)
|
||||
|
||||
monkeypatch.setattr(web_capture, "fetch", fake)
|
||||
return asked
|
||||
|
||||
|
||||
def _fetch(url=None, html=None, source_url=None, name=None):
|
||||
return raw_fetch_command(url=url, html=html, source_url=source_url, name=name)
|
||||
|
||||
|
||||
def _refused(**kwargs) -> None:
|
||||
with pytest.raises(typer.Exit) as excinfo:
|
||||
_fetch(**kwargs)
|
||||
assert excinfo.value.exit_code == 1
|
||||
|
||||
|
||||
def _md(path: Path) -> tuple[dict, str]:
|
||||
head, _, text = path.read_text(encoding="utf-8")[4:].partition("\n---\n")
|
||||
return yaml.safe_load(head), text
|
||||
|
||||
|
||||
def _checks(capsys) -> list[str]:
|
||||
return [line[len("CHECK "):] for line in capsys.readouterr().out.splitlines() if line.startswith("CHECK ")]
|
||||
|
||||
|
||||
# --- the Learn handlers ------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_page_furniture_is_dropped_and_the_article_is_kept(tree, served):
|
||||
_fetch(MODULE_URL, name="oslicense")
|
||||
_, text = _md(tree / "incoming/oslicense.md")
|
||||
|
||||
for chrome in CHROME:
|
||||
assert chrome not in text, chrome
|
||||
assert "# OSLicense Module" in text
|
||||
assert "## OSLicense" in text
|
||||
for cmdlet in CMDLETS:
|
||||
assert f"| [{cmdlet}](https://learn.microsoft.com/en-us/powershell/module/oslicense/{cmdlet.lower()}?view=windowsserver2025-ps) |" in text
|
||||
|
||||
|
||||
def test_the_same_page_of_another_host_keeps_its_furniture(tree, served):
|
||||
_fetch("https://example.org/page", name="other")
|
||||
header, text = _md(tree / "incoming/other.md")
|
||||
|
||||
for chrome in ("requires authorization", "Exit editor mode", "Add to Plans"):
|
||||
assert chrome in text
|
||||
assert "site" not in header and "requested_url" not in header
|
||||
|
||||
|
||||
def test_the_html_is_stored_exactly_as_the_server_sent_it(tree, served):
|
||||
_fetch(MODULE_URL, name="oslicense")
|
||||
assert (tree / "incoming/oslicense.html").read_bytes() == PAGE
|
||||
|
||||
|
||||
def test_the_standard_form_is_fetched_and_the_header_names_both_urls(tree, served):
|
||||
_fetch(MODULE_URL, name="oslicense")
|
||||
header, _ = _md(tree / "incoming/oslicense.md")
|
||||
|
||||
assert served == [STANDARD_URL]
|
||||
assert header["url"] == STANDARD_URL
|
||||
assert header["requested_url"] == MODULE_URL
|
||||
assert header["site"] == "learn-microsoft-powershell-module"
|
||||
assert header["fetched_by"] == "wikitool raw fetch"
|
||||
|
||||
|
||||
def test_a_url_already_in_standard_form_has_no_requested_url(tree, served):
|
||||
_fetch(STANDARD_URL, name="oslicense")
|
||||
header, _ = _md(tree / "incoming/oslicense.md")
|
||||
|
||||
assert served == [STANDARD_URL]
|
||||
assert "requested_url" not in header
|
||||
assert header["site"] == "learn-microsoft-powershell-module"
|
||||
|
||||
|
||||
def test_the_stem_comes_from_the_standard_form(tree, served):
|
||||
_fetch("https://learn.microsoft.com/de-de/powershell/module/oslicense/get-oslicenseinfo?view=x")
|
||||
assert (tree / "incoming/get-oslicenseinfo.md").is_file()
|
||||
|
||||
|
||||
def test_the_checks_name_the_source_and_the_cmdlet_pages(tree, served, capsys):
|
||||
_fetch(MODULE_URL, name="oslicense")
|
||||
lines = _checks(capsys)
|
||||
|
||||
assert len(lines) == 2
|
||||
assert SOURCE_URL in lines[0]
|
||||
assert "11 cmdlet page(s)" in lines[1]
|
||||
|
||||
|
||||
def test_a_page_of_the_general_handler_has_only_the_source_check(tree, served, capsys):
|
||||
_fetch("https://learn.microsoft.com/de-de/azure/overview?view=x", name="azure")
|
||||
header, text = _md(tree / "incoming/azure.md")
|
||||
lines = _checks(capsys)
|
||||
|
||||
assert header["site"] == "learn-microsoft"
|
||||
assert len(lines) == 1 and SOURCE_URL in lines[0]
|
||||
assert "Exit editor mode" not in text
|
||||
|
||||
|
||||
def test_a_page_without_the_meta_tag_has_no_source_check(tree, monkeypatch, capsys):
|
||||
body = PAGE.replace(b'name="original_content_git_url"', b'name="something_else"')
|
||||
monkeypatch.setattr(
|
||||
web_capture, "fetch",
|
||||
lambda url, ua, timeout=0, max_bytes=0: web_capture.Response(url, url, 200, "text/html", body),
|
||||
)
|
||||
_fetch("https://learn.microsoft.com/en-us/azure/overview", name="azure")
|
||||
assert _checks(capsys) == []
|
||||
|
||||
|
||||
def test_two_runs_over_the_same_bytes_give_the_same_text(tree, served):
|
||||
_fetch(MODULE_URL, name="first")
|
||||
_fetch(MODULE_URL, name="second")
|
||||
|
||||
def stable(name: str) -> list[str]:
|
||||
lines = (tree / f"incoming/{name}.md").read_text(encoding="utf-8").splitlines()
|
||||
return [line.replace(name, "X") for line in lines if not line.startswith("retrieved:")]
|
||||
|
||||
assert stable("first") == stable("second")
|
||||
|
||||
|
||||
# --- --html -----------------------------------------------------------------
|
||||
|
||||
|
||||
def test_html_applies_the_rules_but_keeps_the_url_as_typed(tree, monkeypatch, capsys):
|
||||
def no_network(*args, **kwargs):
|
||||
raise AssertionError("--html must not touch the network")
|
||||
|
||||
monkeypatch.setattr(web_capture, "fetch", no_network)
|
||||
saved = tree / "incoming/oslicense.html"
|
||||
saved.write_bytes(PAGE)
|
||||
_fetch(html=saved, source_url=MODULE_URL)
|
||||
|
||||
header, text = _md(tree / "incoming/oslicense.md")
|
||||
assert saved.read_bytes() == PAGE
|
||||
assert header["url"] == MODULE_URL
|
||||
assert "requested_url" not in header
|
||||
assert header["site"] == "learn-microsoft-powershell-module"
|
||||
assert "Exit editor mode" not in text and "# OSLicense Module" in text
|
||||
assert len(_checks(capsys)) == 2
|
||||
|
||||
|
||||
# --- the instance's part -----------------------------------------------------
|
||||
|
||||
|
||||
def test_a_disabled_handler_does_not_apply(tree, served, monkeypatch, capsys):
|
||||
write_settings(tree, monkeypatch, sites={
|
||||
"disabled": ["learn-microsoft", "learn-microsoft-powershell-module"],
|
||||
})
|
||||
_fetch(MODULE_URL, name="oslicense")
|
||||
header, text = _md(tree / "incoming/oslicense.md")
|
||||
|
||||
assert served == [MODULE_URL]
|
||||
assert header["url"] == MODULE_URL
|
||||
assert "site" not in header and "requested_url" not in header
|
||||
assert "Exit editor mode" in text
|
||||
assert _checks(capsys) == []
|
||||
|
||||
|
||||
def test_switching_off_the_special_handler_leaves_the_general_one(tree, served, monkeypatch, capsys):
|
||||
write_settings(tree, monkeypatch, sites={"disabled": ["learn-microsoft-powershell-module"]})
|
||||
_fetch(MODULE_URL, name="oslicense")
|
||||
header, _ = _md(tree / "incoming/oslicense.md")
|
||||
|
||||
assert header["site"] == "learn-microsoft"
|
||||
assert [SOURCE_URL in line for line in _checks(capsys)] == [True]
|
||||
|
||||
|
||||
def test_the_locale_is_a_setting(tree, served, monkeypatch):
|
||||
write_settings(tree, monkeypatch, sites={"locale": "de-de"})
|
||||
_fetch("https://learn.microsoft.com/en-us/azure/overview", name="azure")
|
||||
assert served == ["https://learn.microsoft.com/de-de/azure/overview"]
|
||||
|
||||
|
||||
def test_an_instance_note_is_printed_as_a_check(tree, served, monkeypatch, capsys):
|
||||
write_settings(tree, monkeypatch, sites={"notes": [
|
||||
{"host": "learn.microsoft.com", "path": "^/en-us/powershell/", "lines": ["First note.", "Second note."]},
|
||||
{"host": "learn.microsoft.com", "path": "^/en-us/azure/", "lines": ["Azure only."]},
|
||||
]})
|
||||
_fetch(MODULE_URL, name="oslicense")
|
||||
lines = _checks(capsys)
|
||||
|
||||
assert lines[-2:] == ["First note.", "Second note."]
|
||||
assert "Azure only." not in lines
|
||||
|
||||
|
||||
def test_a_note_applies_to_a_host_without_a_handler_and_to_html(tree, served, monkeypatch, capsys):
|
||||
write_settings(tree, monkeypatch, sites={"notes": [{"host": "example.org", "lines": ["Mind the paywall."]}]})
|
||||
_fetch("https://example.org/page", name="page")
|
||||
assert _checks(capsys) == ["Mind the paywall."]
|
||||
|
||||
saved = tree / "incoming/saved.html"
|
||||
saved.write_bytes(PAGE)
|
||||
_fetch(html=saved, source_url="https://example.org/other")
|
||||
assert _checks(capsys) == ["Mind the paywall."]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sites_section", [
|
||||
{"disabled": ["no-such-handler"]},
|
||||
{"locale": "Deutsch"},
|
||||
{"notes": [{"host": "example.org", "lines": ["1", "2", "3", "4"]}]},
|
||||
{"notes": [{"host": "example.org", "lines": ["a\nb"]}]},
|
||||
{"notes": [{"host": "Example.org", "lines": ["a"]}]},
|
||||
{"notes": [{"host": "example.org", "path": "(", "lines": ["a"]}]},
|
||||
{"notes": [{"host": "example.org", "lines": ["a"]}, {"host": "example.org", "lines": ["b"]}]},
|
||||
{"notes": [{"host": "example.org", "lines": ["a"], "extra": 1}]},
|
||||
])
|
||||
def test_a_broken_sites_section_stops_before_anything_is_fetched(tree, served, monkeypatch, sites_section):
|
||||
write_settings(tree, monkeypatch, sites=sites_section)
|
||||
_refused(url="https://example.org/page")
|
||||
assert served == []
|
||||
assert list((tree / "incoming").iterdir()) == []
|
||||
|
||||
|
||||
# --- the registry ------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry(monkeypatch):
|
||||
monkeypatch.setattr(sites, "_REGISTRY", list(sites._REGISTRY))
|
||||
return sites._REGISTRY
|
||||
|
||||
|
||||
def test_the_longest_path_pattern_wins():
|
||||
url = "https://learn.microsoft.com/en-us/powershell/module/x/"
|
||||
assert sites.site_for(url).name == "learn-microsoft-powershell-module"
|
||||
assert sites.site_for("https://learn.microsoft.com/en-us/powershell/module/x/y").name == "learn-microsoft"
|
||||
assert sites.site_for("https://Learn.Microsoft.com/en-us/azure").name == "learn-microsoft"
|
||||
assert sites.site_for("https://example.org/en-us/powershell/module/x/") is None
|
||||
|
||||
|
||||
def test_a_special_handler_inherits_from_the_general_one():
|
||||
general = next(s for s in sites.all_sites() if s.name == "learn-microsoft")
|
||||
special = next(s for s in sites.all_sites() if s.name == "learn-microsoft-powershell-module")
|
||||
assert special.drop == general.drop
|
||||
assert special.rewrites == general.rewrites
|
||||
assert special.checks[: len(general.checks)] == general.checks
|
||||
assert len(special.checks) == len(general.checks) + 1
|
||||
|
||||
|
||||
def test_a_duplicate_host_and_path_is_a_registration_error(registry):
|
||||
with pytest.raises(ValueError, match="same host and path pattern"):
|
||||
sites.register(sites.Site(name="again", host="learn.microsoft.com", summary="x"))
|
||||
|
||||
|
||||
def test_a_duplicate_name_is_a_registration_error(registry):
|
||||
with pytest.raises(ValueError, match="already registered"):
|
||||
sites.register(sites.Site(name="learn-microsoft", host="example.org", summary="x"))
|
||||
|
||||
|
||||
def test_a_duplicate_in_a_module_fails_at_import(registry, tmp_path, monkeypatch):
|
||||
(tmp_path / "duplicate.py").write_text(
|
||||
"from chemenu.sites import Site, register\n"
|
||||
"register(Site(name='dup', host='learn.microsoft.com', summary='x'))\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(sites, "__path__", [*sites.__path__, str(tmp_path)])
|
||||
monkeypatch.delitem(sys.modules, "chemenu.sites.duplicate", raising=False)
|
||||
with pytest.raises(ValueError, match="same host and path pattern"):
|
||||
importlib.import_module("chemenu.sites.duplicate")
|
||||
|
||||
|
||||
def test_a_parent_must_exist_and_share_the_host(registry):
|
||||
with pytest.raises(ValueError, match="not registered"):
|
||||
sites.register(sites.Site(name="child", host="example.org", summary="x", parent="nobody"))
|
||||
with pytest.raises(ValueError, match="another host"):
|
||||
sites.register(sites.Site(name="child", host="example.org", summary="x", parent="learn-microsoft"))
|
||||
|
||||
|
||||
def test_a_handler_has_at_most_three_checks():
|
||||
with pytest.raises(ValueError, match="at most 3"):
|
||||
sites.Site(name="n", host="example.org", summary="x", checks=(sites.Check("a"),) * 4)
|
||||
|
||||
|
||||
def test_an_inherited_handler_stays_within_the_limit(registry):
|
||||
sites.register(sites.Site(name="base", host="example.org", summary="x", checks=(sites.Check("a"),) * 3))
|
||||
with pytest.raises(ValueError, match="at most 3"):
|
||||
sites.register(sites.Site(name="sub", host="example.org", summary="x", path="/a/", parent="base",
|
||||
checks=(sites.Check("b"),)))
|
||||
|
||||
|
||||
# --- the building blocks -----------------------------------------------------
|
||||
|
||||
|
||||
def test_a_rule_needs_something_to_match_on():
|
||||
with pytest.raises(ValueError):
|
||||
web_capture.Rule()
|
||||
with pytest.raises(ValueError):
|
||||
web_capture.Rule(tag="a", inside="td", value="x")
|
||||
|
||||
|
||||
def test_rules_match_by_tag_id_class_attribute_and_ancestor():
|
||||
document = web_capture.parse(
|
||||
'<div id="a" class="x y"><p class="y" data-k="v">one</p></div><table><tr><td><a>two</a></td></tr></table><a>three</a>'
|
||||
)
|
||||
count = web_capture.count_matches
|
||||
assert count(document, web_capture.Rule(id="a")) == 1
|
||||
assert count(document, web_capture.Rule(cls="y")) == 2
|
||||
assert count(document, web_capture.Rule(cls="x y")) == 0
|
||||
assert count(document, web_capture.Rule(attr="data-k")) == 1
|
||||
assert count(document, web_capture.Rule(attr="data-k", value="w")) == 0
|
||||
assert count(document, web_capture.Rule(tag="a")) == 2
|
||||
assert count(document, web_capture.Rule(tag="a", inside="td")) == 1
|
||||
|
||||
|
||||
def test_derive_text_without_rules_is_unchanged():
|
||||
html = "<main><h1>T</h1><p>x</p><div id='gone'>y</div></main>"
|
||||
assert web_capture.derive_text(html, "https://e.org/") == "# T\n\nx\n\ny"
|
||||
assert web_capture.derive_text(html, "https://e.org/", (web_capture.Rule(id="gone"),)) == "# T\n\nx"
|
||||
|
||||
|
||||
def test_a_rewrite_is_a_regular_expression_with_the_locale_filled_in():
|
||||
step = sites.Rewrite(r"^(https://e\.org/)[a-z-]+(?=/)", r"\1{locale}")
|
||||
assert step.apply("https://e.org/de-de/a/b?view=1", "fr-fr") == "https://e.org/fr-fr/a/b?view=1"
|
||||
Reference in new issue
Block a user