Files
chemenu/tools/chemenu/tests/test_provenance.py
T
torben f63a72cb24
CI / verify (push) Successful in 42s
Release / release (push) Successful in 37s
fix: raw_dir-Fixture kappt config.ROOT, private Fixture-Namen ersetzt (2.1.1)
Files changed:
- CHANGES.md
- VERSION
- tools/chemenu/tests/conftest.py
- tools/chemenu/tests/test_corpus_diff.py
- tools/chemenu/tests/test_index_build.py
- tools/chemenu/tests/test_kb_collections.py
- tools/chemenu/tests/test_lint.py
- tools/chemenu/tests/test_migrate_cmd.py
- tools/chemenu/tests/test_new_page.py
- tools/chemenu/tests/test_page_ops.py
- tools/chemenu/tests/test_provenance.py
- tools/chemenu/tests/test_search.py
- tools/chemenu/tests/test_touch.py
2026-09-01 17:03:51 +02:00

550 lines
20 KiB
Python

from pathlib import Path
from typer.testing import CliRunner
from chemenu.frontmatter_io import read_page, write_page
from chemenu.provenance import (
broken_raw_refs,
cite_id,
citing_pages,
duplicate_raw_file_owners,
extract_inline_cites,
legacy_citation_markers,
legacy_source_pages,
page_raw_files,
render_cite_block,
render_page_body,
source_pages_by_raw_file,
source_raw_files,
split_cite_block,
unique_cite_id,
uncovered_raw_files,
)
from chemenu.kb_scan import load_kb_pages
runner = CliRunner()
def _footnote_block(*cites: tuple[str, str | None]) -> str:
"""Build a `[^id]` reference for each (title, qualifier) plus the
trailing Footnotes block defining it, in one string - the fixture form
tests use in place of the old `^[[Title]]` marker."""
ids = [cite_id(title, qualifier) for title, qualifier in cites]
refs = "".join(f"[^{cid}]" for cid in ids)
block = render_cite_block({cid: cite for cid, cite in zip(ids, cites)})
return refs, block
def _source_page(title: str, raw_files: list[str]):
from chemenu.page import Page
return Page(
path=Path(f"/tmp/{title}.md"),
frontmatter={"type": "types/source.md", "raw_files": raw_files},
body="",
)
def test_duplicate_raw_file_owners_reports_a_file_claimed_twice():
"""The Almanac 10-bootstrap-manual case: an umbrella page and a per-step page
both claiming the same raw file, which uncovered_raw_files() cannot see."""
pages = {
"Source - Umbrella": _source_page("Source - Umbrella", ["raw/a.md", "raw/b.md"]),
"Source - Step": _source_page("Source - Step", ["raw/b.md"]),
}
assert duplicate_raw_file_owners(pages) == [
{"raw_file": "raw/b.md", "owners": ["Source - Step", "Source - Umbrella"]}
]
def test_duplicate_raw_file_owners_is_empty_when_ownership_is_unique():
pages = {
"Source - A": _source_page("Source - A", ["raw/a.md"]),
"Source - B": _source_page("Source - B", ["raw/b.md"]),
}
assert duplicate_raw_file_owners(pages) == []
def test_duplicate_raw_file_owners_ignores_repeats_within_one_page():
"""A file listed twice in the same page's raw_files: is untidy, not a
contested claim - there is still exactly one owner."""
pages = {"Source - A": _source_page("Source - A", ["raw/a.md", "raw/a.md"])}
assert duplicate_raw_file_owners(pages) == []
def test_extract_inline_cites_plain_and_qualified():
refs, block = _footnote_block(
("Source - Aurora", None), ("Source - Almanac Architecture", "storage-model.md")
)
body = f"Fact one {refs}. Fact two {refs}.\n\n{block}"
cites = extract_inline_cites(body)
assert ("Source - Aurora", None) in cites
assert ("Source - Almanac Architecture", "storage-model.md") in cites
def test_extract_inline_cites_ignores_plain_wikilinks():
body = "See [[Source - Aurora]] for background, but this line has no hard fact."
assert extract_inline_cites(body) == set()
def test_extract_inline_cites_ignores_undefined_ref():
"""A `[^id]` with no matching definition resolves to nothing here - lint's
undefined_footnote_refs is what flags that, not extract_inline_cites."""
assert extract_inline_cites("Fact one [^s-ghost].") == set()
def test_cite_id_strips_prefix_and_slugifies():
assert cite_id("Source - Aurora") == "s-aurora"
assert cite_id("Source - Almanac Architecture", "storage-model.md") == "s-almanac-architecture--storage-model-md"
def test_cite_id_is_deterministic_and_ascii():
assert cite_id("Source - Würfelspiel für Fortgeschrittene") == cite_id(
"Source - Würfelspiel für Fortgeschrittene"
)
assert cite_id("Source - Würfelspiel für Fortgeschrittene").isascii()
def test_unique_cite_id_suffixes_on_collision():
base = cite_id("Source - Aurora")
assert unique_cite_id(set(), "Source - Aurora") == base
assert unique_cite_id({base}, "Source - Aurora") == f"{base}-2"
assert unique_cite_id({base, f"{base}-2"}, "Source - Aurora") == f"{base}-3"
def test_split_and_render_cite_block_round_trip():
definitions = {"s-aurora": ("Source - Aurora", None), "s-almanac--x-md": ("Source - Almanac", "x.md")}
body = "# Page\n\nSome prose [^s-aurora].\n\n" + render_cite_block(definitions)
head, parsed = split_cite_block(body)
assert parsed == definitions
assert head == "# Page\n\nSome prose [^s-aurora]."
def test_split_cite_block_empty_when_no_footnotes_heading():
head, definitions = split_cite_block("# Page\n\nNo citations here.\n")
assert definitions == {}
assert head == "# Page\n\nNo citations here."
# --- content after the Footnotes block ---------------------------------------
#
# The block used to run to the end of the file, so a section sitting after it
# was deleted on the next cite add / cite sync / rename. `xref add` appends its
# sections at the end of the file, so which command ran last decided whether a
# page kept its cross-references.
def test_a_section_after_the_block_survives_the_round_trip():
definitions = {"s-aurora": ("Source - Aurora", None)}
relationships = "## Beziehungen\n\n- **umgesetzt von:** [[wikitool]]\n"
body = (
"# Page\n\nSome prose [^s-aurora].\n\n"
+ render_cite_block(definitions)
+ "\n"
+ relationships
)
head, parsed = split_cite_block(body)
assert parsed == definitions
assert "## Beziehungen" in head
assert "[[wikitool]]" in head
rebuilt = render_page_body(head, parsed)
assert "- **umgesetzt von:** [[wikitool]]" in rebuilt
assert "[^s-aurora]: [[Source - Aurora]]" in rebuilt
def test_the_block_is_re_emitted_last_so_the_layout_self_heals():
"""`xref add` appends at the end of the file. Folding the rescued tail into
the head means the next cite operation puts the block back at the bottom
instead of preserving the broken order forever."""
definitions = {"s-aurora": ("Source - Aurora", None)}
body = (
"# Page\n\nProse [^s-aurora].\n\n"
+ render_cite_block(definitions)
+ "\n## Siehe auch\n\n- [[Borealis]]\n"
)
rebuilt = render_page_body(*split_cite_block(body))
assert rebuilt.index("## Siehe auch") < rebuilt.index("[^s-aurora]:")
def test_repeated_round_trips_are_stable():
"""Rescuing content must not move it a little further on every run."""
definitions = {"s-aurora": ("Source - Aurora", None)}
body = (
"# Page\n\nProse [^s-aurora].\n\n"
+ render_cite_block(definitions)
+ "\n## Siehe auch\n\n- [[Borealis]]\n"
)
once = render_page_body(*split_cite_block(body))
twice = render_page_body(*split_cite_block(once))
assert once == twice
def test_stray_prose_inside_the_block_is_kept_not_dropped():
"""Not a definition and not a section - rescued rather than rejected,
because this runs under lint too, where raising would refuse to read a
page instead of reporting it."""
body = (
"# Page\n\nProse [^s-aurora].\n\n"
"## Fußnoten\n\n"
"[^s-aurora]: [[Source - Aurora]]\n"
"TODO: check this one\n"
)
head, definitions = split_cite_block(body)
assert definitions == {"s-aurora": ("Source - Aurora", None)}
assert "TODO: check this one" in head
def test_a_citation_used_in_a_rescued_section_still_resolves():
body = (
"# Page\n\nProse.\n\n"
"## Fußnoten\n\n"
"[^s-aurora]: [[Source - Aurora]]\n\n"
"## Beziehungen\n\n- **belegt durch:** [[Borealis]] [^s-aurora]\n"
)
assert extract_inline_cites(body) == {("Source - Aurora", None)}
def test_source_raw_files_prefers_raw_files_over_legacy_source():
from chemenu.page import Page
page = Page(
path=Path("/tmp/Source - X.md"),
frontmatter={"type": "source", "raw_files": ["raw/notes/A.md"], "source": "https://example.com"},
body="",
)
assert source_raw_files(page) == ["raw/notes/A.md"]
def test_source_raw_files_falls_back_to_legacy_repo_path():
from chemenu.page import Page
page = Page(
path=Path("/tmp/Source - Y.md"),
frontmatter={"type": "source", "source": "raw/notes/B.md"},
body="",
)
assert source_raw_files(page) == ["raw/notes/B.md"]
def test_source_raw_files_ignores_url_only_legacy_source():
from chemenu.page import Page
page = Page(
path=Path("/tmp/Source - Z.md"),
frontmatter={"type": "source", "source": "https://example.com/article"},
body="",
)
assert source_raw_files(page) == []
def test_source_pages_by_raw_file_uses_legacy_source_fallback(kb_dir, raw_dir):
pages = load_kb_pages(kb_dir)
by_raw = source_pages_by_raw_file(pages)
assert by_raw.get("raw/notes/Aurora.md") == ["Source - Aurora"]
def test_uncovered_raw_files_detects_ingested_gap(kb_dir, raw_dir, monkeypatch):
import chemenu.config as config
monkeypatch.setattr(config, "ROOT", raw_dir.parent)
pages = load_kb_pages(kb_dir)
uncovered = uncovered_raw_files(raw_dir, pages)
assert "raw/notes/Uningested.md" in uncovered
assert "raw/notes/Aurora.md" not in uncovered
def test_broken_raw_refs_detects_dangling_path(kb_dir, raw_dir, monkeypatch):
import chemenu.config as config
monkeypatch.setattr(config, "ROOT", raw_dir.parent)
write_page(
kb_dir / "sources/Source - Ghost.md",
{
"type": "source", "source_type": "notes", "author": "Torben",
"raw_files": ["raw/notes/Does Not Exist.md"], "date": "2026-08-02",
"tags": [], "entities": [], "concepts": [],
},
"\n# Source: Ghost\n\n## Summary\n\nGhost.\n",
)
pages = load_kb_pages(kb_dir)
issues = broken_raw_refs(pages)
assert {"page": "Source - Ghost", "raw_path": "raw/notes/Does Not Exist.md"} in issues
def test_legacy_source_pages_flags_url_and_directory(kb_dir, raw_dir):
write_page(
kb_dir / "sources/Source - External.md",
{
"type": "source", "source_type": "article", "author": "someone",
"source": "https://example.com/article", "date": "2026-08-02",
"tags": [], "entities": [], "concepts": [],
},
"\n# Source: External\n\n## Summary\n\nExternal.\n",
)
(raw_dir / "documents").mkdir()
write_page(
kb_dir / "sources/Source - DirBacked.md",
{
"type": "source", "source_type": "document", "author": "Torben",
"source": "raw/documents", "date": "2026-08-02",
"tags": [], "entities": [], "concepts": [],
},
"\n# Source: DirBacked\n\n## Summary\n\nDir.\n",
)
pages = load_kb_pages(kb_dir)
issues = legacy_source_pages(pages)
reasons = {i["page"]: i["reason"] for i in issues}
assert reasons["Source - External"] == "url-only, no raw_files"
assert reasons["Source - DirBacked"] == "directory, not a file"
def test_citing_pages_via_frontmatter_and_inline(kb_dir, raw_dir):
write_page(
kb_dir / "entities/tools/gdeploy.md",
{
"type": "entity", "entity_type": "tool", "tags": [], "created": "2026-07-25",
"modified": "2026-07-25", "related": [], "sources": ["Source - Aurora"], "confidence": 0.8,
},
"\n# gdeploy\n\n## Description\n\nDeploy tool.\n",
)
refs, block = _footnote_block(("Source - Aurora", None))
write_page(
kb_dir / "concepts/Modbus.md",
{
"type": "concept", "concept_type": "protocol", "tags": [], "created": "2026-07-25",
"modified": "2026-07-25", "related": [], "sources": [], "confidence": 0.7,
},
f"\n# Modbus\n\n## Definition\n\nUses port 502 {refs}.\n\n{block}",
)
pages = load_kb_pages(kb_dir)
citers = citing_pages(pages, "Source - Aurora")
assert "gdeploy" in citers
assert "Modbus" in citers
def test_page_raw_files_resolves_through_sources_and_inline(kb_dir, raw_dir):
write_page(
kb_dir / "concepts/Modbus.md",
{
"type": "concept", "concept_type": "protocol", "tags": [], "created": "2026-07-25",
"modified": "2026-07-25", "related": [], "sources": ["Source - Aurora"], "confidence": 0.7,
},
"\n# Modbus\n\n## Definition\n\nIndustrial protocol.\n",
)
pages = load_kb_pages(kb_dir)
raw_files = page_raw_files(pages, pages["Modbus"])
assert raw_files == ["raw/notes/Aurora.md"]
def test_provenance_md_is_not_a_wiki_page(kb_dir):
(kb_dir / "provenance.md").write_text("# Provenance Index\n", encoding="utf-8")
pages = load_kb_pages(kb_dir)
assert "provenance" not in pages
def test_sources_coverage_command_reports_uncovered_file(kb_dir, raw_dir, monkeypatch):
import chemenu.config as config
from chemenu.cli import app
monkeypatch.setattr(config, "ROOT", raw_dir.parent)
monkeypatch.setattr(config, "KB_DIR", kb_dir)
monkeypatch.setattr(config, "RAW_DIR", raw_dir)
result = runner.invoke(app, ["sources", "coverage", "--json"])
assert result.exit_code == 0, result.output
assert "raw/notes/Uningested.md" in result.output
def test_sources_trace_by_raw_and_by_page(kb_dir, raw_dir, monkeypatch):
import chemenu.config as config
from chemenu.cli import app
monkeypatch.setattr(config, "ROOT", raw_dir.parent)
monkeypatch.setattr(config, "KB_DIR", kb_dir)
monkeypatch.setattr(config, "RAW_DIR", raw_dir)
write_page(
kb_dir / "entities/systems/aurora.md",
{
"type": "entity", "entity_type": "system", "tags": ["server"],
"created": "2026-07-31", "modified": "2026-07-31", "related": ["Borealis"],
"sources": ["Source - Aurora"], "confidence": 0.9,
},
"\n# aurora\n\n## Description\n\nHosts things.\n",
)
result = runner.invoke(app, ["sources", "trace", "--raw", "raw/notes/Aurora.md"])
assert result.exit_code == 0, result.output
assert "Source - Aurora" in result.output
assert "aurora" in result.output
result2 = runner.invoke(app, ["sources", "trace", "--page", "aurora"])
assert result2.exit_code == 0, result2.output
assert "Source - Aurora" in result2.output
assert "raw/notes/Aurora.md" in result2.output
def test_new_source_with_multiple_raw_files(kb_dir, raw_dir, monkeypatch):
import chemenu.config as config
from chemenu.cli import app
monkeypatch.setattr(config, "ROOT", raw_dir.parent)
monkeypatch.setattr(config, "KB_DIR", kb_dir)
# This test is about raw_files:, not about authorship - but `new source`
# refuses to stamp a placeholder author, and the fixture root has no git
# identity. Naming one keeps the test off the runner's global git config.
monkeypatch.setenv("WIKI_AUTHOR", "Fixture Author")
(raw_dir / "notes" / "Second.md").write_text("# Second\n", encoding="utf-8")
result = runner.invoke(app, [
"new", "source", "--name", "Multi",
"--set", "raw_files=raw/notes/Aurora.md,raw/notes/Second.md",
])
assert result.exit_code == 0, result.output
fm, _body = read_page(kb_dir / "sources/Source - Multi.md")
assert fm["raw_files"] == ["raw/notes/Aurora.md", "raw/notes/Second.md"]
def test_new_source_rejects_nonexistent_raw_path(kb_dir, raw_dir, monkeypatch):
import chemenu.config as config
from chemenu.cli import app
monkeypatch.setattr(config, "ROOT", raw_dir.parent)
monkeypatch.setattr(config, "KB_DIR", kb_dir)
result = runner.invoke(app, [
"new", "source", "--name", "Bad", "--set", "raw_files=raw/notes/Nope.md",
])
assert result.exit_code != 0
def test_lint_flags_citation_not_in_frontmatter_sources(kb_dir, raw_dir, monkeypatch):
import chemenu.config as config
from chemenu.commands.lint import run_lint
monkeypatch.setattr(config, "ROOT", raw_dir.parent)
monkeypatch.setattr(config, "RAW_DIR", raw_dir)
refs, block = _footnote_block(("Source - Aurora", None))
write_page(
kb_dir / "concepts/Modbus.md",
{
"type": "concept", "concept_type": "protocol", "tags": [], "created": "2026-07-25",
"modified": "2026-07-25", "related": [], "sources": [], "confidence": 0.7,
"provenance": "sourced",
},
f"\n# Modbus\n\n## Definition\n\nUses port 502 {refs}.\n\n{block}",
)
report = run_lint(kb_dir)
drift = report["citation_frontmatter_drift"]
assert {"page": "Modbus", "cited_but_not_in_sources": "Source - Aurora"} in drift
def test_lint_no_drift_when_source_declared(kb_dir, raw_dir, monkeypatch):
import chemenu.config as config
from chemenu.commands.lint import run_lint
monkeypatch.setattr(config, "ROOT", raw_dir.parent)
monkeypatch.setattr(config, "RAW_DIR", raw_dir)
refs, block = _footnote_block(("Source - Aurora", None))
write_page(
kb_dir / "concepts/Modbus.md",
{
"type": "concept", "concept_type": "protocol", "tags": [], "created": "2026-07-25",
"modified": "2026-07-25", "related": [], "sources": ["Source - Aurora"], "confidence": 0.7,
"provenance": "sourced",
},
f"\n# Modbus\n\n## Definition\n\nUses port 502 {refs}.\n\n{block}",
)
report = run_lint(kb_dir)
assert report["citation_frontmatter_drift"] == []
def test_lint_unmarked_provenance_flags_empty_sources_without_general(kb_dir, raw_dir, monkeypatch):
import chemenu.config as config
from chemenu.commands.lint import run_lint
monkeypatch.setattr(config, "ROOT", raw_dir.parent)
monkeypatch.setattr(config, "RAW_DIR", raw_dir)
report = run_lint(kb_dir)
# gdeploy in the base fixture has sources: [] and no provenance field at all.
assert "gdeploy" in report["unmarked_provenance"]
def test_lint_does_not_flag_source_page_self_citation(kb_dir, raw_dir, monkeypatch):
import chemenu.config as config
from chemenu.commands.lint import run_lint
monkeypatch.setattr(config, "ROOT", raw_dir.parent)
monkeypatch.setattr(config, "RAW_DIR", raw_dir)
refs, block = _footnote_block(("Source - Aurora", "Aurora.md"))
write_page(
kb_dir / "sources/Source - Aurora.md",
{
"type": "source", "source_type": "notes", "author": "Torben",
"raw_files": ["raw/notes/Aurora.md"], "date": "2026-08-02",
"tags": [], "entities": ["aurora"], "concepts": [],
},
f"\n# Source: Aurora\n\n## Summary\n\nCovers ZFS setup {refs}.\n\n{block}",
)
report = run_lint(kb_dir)
assert not any(i["page"] == "Source - Aurora" for i in report["citation_frontmatter_drift"])
def _page(body: str):
from chemenu.page import Page
return Page(path=Path("/tmp/Notation.md"), frontmatter={"type": "types/concept.md"}, body=body)
def test_extract_inline_cites_ignores_notation_shown_as_code():
"""A page describing the citation mechanism resolves nothing: the markers
it shows are examples. This also decides what counts as sourced, so a
mention used to be able to raise a page's apparent provenance."""
block = render_cite_block({
"s-real": ("Source - Real", None),
"s-shown": ("Source - Shown", None),
})
body = (
"The marker `[^s-shown]` is pasted at the fact [^s-real].\n\n"
"```markdown\n[^s-shown]: [[Source - Shown]]\n```\n\n" + block
)
# `s-shown` is defined and appears twice - in backticks and in a fence -
# and is cited by neither. Only the reference in prose resolves.
assert extract_inline_cites(body) == {("Source - Real", None)}
def test_split_cite_block_ignores_a_fenced_definition_inside_the_block():
"""The counter-direction: a definition line shown as an example does not
become a definition, which would then read as an orphan."""
body = (
"# Notation\n\nProse [^s-real].\n\n"
"## Fußnoten\n\n"
"[^s-real]: [[Source - Real]]\n\n"
"```markdown\n[^s-example]: [[Source - Example]]\n```\n"
)
head, definitions = split_cite_block(body)
assert definitions == {"s-real": ("Source - Real", None)}
assert "[^s-example]" in head # rescued, not discarded
def test_legacy_citation_markers_ignore_notation_shown_as_code():
"""The pre-migration marker is exactly the thing a page about the
migration has to be able to quote."""
pages = {
"Notation": _page(
"The old form was `^[[Source - X]]`, replaced by a footnote:\n\n"
"```markdown\n^[[Source - Y]]\n```\n"
)
}
assert legacy_citation_markers(pages) == []
def test_legacy_citation_markers_still_flag_a_real_one():
pages = {"Stale": _page("A fact ^[[Source - X]] that was never migrated.\n")}
assert legacy_citation_markers(pages) == [{"page": "Stale", "marker": "^[[Source - X]]"}]