feat: raw accept - incoming/ als abgeleiteter Rohablage-Eingang (schliesst #58)
Files changed: - .gitignore - CHANGES.md - README.md - VERSION - docs/pipeline-rationale.md - instructions/bootstrap.md - instructions/wiki-ingest/SKILL.md - raw/CONTRACT.md - tools/CONTRACT.md - tools/chemenu/cli.py - tools/chemenu/commands/dist_cmd.py - tools/chemenu/commands/docs_verify.py - tools/chemenu/commands/raw_cmd.py - tools/chemenu/tests/test_dist_cmd.py - tools/chemenu/tests/test_docs_verify.py - tools/chemenu/tests/test_raw_cmd.py
This commit is contained in:
@@ -27,6 +27,7 @@ try:
|
||||
new_page,
|
||||
page_ops,
|
||||
provenance_cmd,
|
||||
raw_cmd,
|
||||
run_budget,
|
||||
search as search_module,
|
||||
touch as touch_module,
|
||||
@@ -61,6 +62,7 @@ app.add_typer(index_build.app, name="index")
|
||||
app.add_typer(log_append.app, name="log")
|
||||
app.add_typer(confidence_decay.app, name="confidence")
|
||||
app.add_typer(provenance_cmd.app, name="sources")
|
||||
app.add_typer(raw_cmd.app, name="raw")
|
||||
app.add_typer(instructions_cmd.app, name="instructions")
|
||||
app.add_typer(run_budget.app, name="budget")
|
||||
app.add_typer(types_cmd.app, name="types")
|
||||
|
||||
@@ -135,6 +135,10 @@ INSTRUCTIONS_EXCLUDE_DIRS = {"dev"}
|
||||
|
||||
# Fixed by raw/CONTRACT.md's routing table, unlike kb/'s areas (which are
|
||||
# organic - see kb/CONTRACT.md - so `export` does not manufacture them).
|
||||
# `docs verify` (check_raw_subdirs) holds the table to this tuple in both
|
||||
# directions, and `incoming/` (raw/CONTRACT.md "Getting a file in",
|
||||
# raw_cmd.py) mirrors it as the set of type subdirectories a human may drop a
|
||||
# file into - so this is the one place all three read the list from.
|
||||
RAW_SUBDIRS = ("articles", "documents", "notes", "assets")
|
||||
|
||||
# Stage contracts that are not collections and carry no pages: copied as a
|
||||
@@ -414,6 +418,12 @@ def build_plan(origin: Optional[Origin] = None) -> dict[str, PlannedFile]:
|
||||
|
||||
for sub in RAW_SUBDIRS:
|
||||
plan[f"raw/{sub}/.gitkeep"] = PlannedFile("")
|
||||
# `incoming/` mirrors raw/'s type subdirectories (raw/CONTRACT.md
|
||||
# "Getting a file in") - seeded the same way, though `.gitignore`
|
||||
# (also exported, see ROOT_FILES) excludes the whole directory again
|
||||
# once the instance is a git repo, which is why bootstrap.md re-creates
|
||||
# it for a plain clone that never had this export step at all.
|
||||
plan[f"incoming/{sub}/.gitkeep"] = PlannedFile("")
|
||||
|
||||
plan["kb/log.md"] = PlannedFile((DIST_TEMPLATES_DIR / "log.md").read_text(encoding="utf-8"))
|
||||
plan["CHANGES.md"] = PlannedFile((DIST_TEMPLATES_DIR / "CHANGES.md").read_text(encoding="utf-8"))
|
||||
|
||||
@@ -38,6 +38,7 @@ from typing import Optional
|
||||
import typer
|
||||
|
||||
from chemenu import config, conventions, kb_collections, version as version_mod
|
||||
from chemenu.commands import dist_cmd
|
||||
from chemenu.commands._util import fail, rel_path, success
|
||||
|
||||
app = typer.Typer(help="Verify documentation that mirrors the code or repo layout.")
|
||||
@@ -108,6 +109,12 @@ REQUIRED_IGNORE_CANARIES = (
|
||||
"ENVIRONMENT.md",
|
||||
"tools/coverage.xml",
|
||||
"tools/htmlcov/index.html",
|
||||
# The ingest inbox (Gitea #58, raw/CONTRACT.md "Getting a file in"). Unlike
|
||||
# raw/ itself, a file here must never be committed - promotion via
|
||||
# `wikitool raw accept` is what makes it immutable, not the drop - so this
|
||||
# is the one canary in this tuple asserting the *opposite* of raw/'s own
|
||||
# backstop a few lines above.
|
||||
"incoming/documents/probe.pdf",
|
||||
)
|
||||
REQUIRED_TRACKED_PATHS = (
|
||||
"reports/CONTRACT.md",
|
||||
@@ -158,6 +165,11 @@ LEGACY_TYPE_RE = re.compile(r"^type:\s*(entity|concept|source|comparison)\s*$",
|
||||
# First backticked cell of a markdown table row, e.g. "| `xref add --a ...` | ... |"
|
||||
TABLE_CELL_RE = re.compile(r"^\|\s*`([^`]+)`", re.MULTILINE)
|
||||
|
||||
# A raw/CONTRACT.md routing-table cell naming a bare type subdirectory, e.g.
|
||||
# "| `articles/` | ... |" - deliberately narrower than TABLE_CELL_RE, which
|
||||
# would also match a command example elsewhere on the page.
|
||||
RAW_DIR_CELL_RE = re.compile(r"^\|\s*`([a-zA-Z0-9_-]+)/`\s*\|", re.MULTILINE)
|
||||
|
||||
|
||||
def registered_commands() -> set[str]:
|
||||
"""Every command path the CLI exposes, e.g. {'new', 'xref add', ...}.
|
||||
@@ -317,6 +329,34 @@ def check_stack_required_types() -> list[str]:
|
||||
return issues
|
||||
|
||||
|
||||
def documented_raw_subdirs(text: str) -> list[str]:
|
||||
return [match.group(1) for match in RAW_DIR_CELL_RE.finditer(text)]
|
||||
|
||||
|
||||
def check_raw_subdirs() -> list[str]:
|
||||
"""`raw/CONTRACT.md`'s routing table and `dist_cmd.RAW_SUBDIRS` must name
|
||||
the same set of type subdirectories (Gitea #58) - the table is meant to
|
||||
read as behaviour derived from the tuple, not as a second place the list
|
||||
could drift (AGENTS.md invariant 8). Skipped if the contract itself is
|
||||
missing; `check_collection_contracts` already reports that.
|
||||
"""
|
||||
contract_path = config.ROOT / "raw" / "CONTRACT.md"
|
||||
if not contract_path.exists():
|
||||
return []
|
||||
documented = set(documented_raw_subdirs(contract_path.read_text(encoding="utf-8")))
|
||||
declared = set(dist_cmd.RAW_SUBDIRS)
|
||||
issues = [
|
||||
f"raw/CONTRACT.md's routing table is missing `{missing}/` - dist_cmd.RAW_SUBDIRS names it"
|
||||
for missing in sorted(declared - documented)
|
||||
]
|
||||
issues += [
|
||||
f"raw/CONTRACT.md's routing table lists `{extra}/`, but dist_cmd.RAW_SUBDIRS does not - "
|
||||
"the two must name the same set"
|
||||
for extra in sorted(documented - declared)
|
||||
]
|
||||
return issues
|
||||
|
||||
|
||||
def check_legacy_type_blocks() -> list[str]:
|
||||
issues = []
|
||||
guarded = [
|
||||
@@ -585,6 +625,7 @@ def verify():
|
||||
check_cli_readme()
|
||||
+ check_readmes_have_no_command_table()
|
||||
+ check_collection_contracts()
|
||||
+ check_raw_subdirs()
|
||||
+ check_legacy_type_blocks()
|
||||
+ check_ignored_content()
|
||||
+ check_version_changelog()
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
"""`wikitool raw accept` - promote one or more files from `incoming/` into
|
||||
`raw/`, with the destination computed rather than chosen by hand (Gitea #58).
|
||||
|
||||
A human classifies a file only by which type subdirectory of `incoming/` they
|
||||
drop it into - `incoming/articles/`, `incoming/documents/`, `incoming/notes/`,
|
||||
`incoming/assets/`, mirroring raw/CONTRACT.md's routing table. Everything past
|
||||
that is this command's job:
|
||||
|
||||
- **Single file, no bundle.** One file promoted alone lands as
|
||||
`raw/<type>/<name>` - no directory of its own.
|
||||
- **Bundle from the second file on.** Several files of one source promoted in
|
||||
the same call land under `raw/<type>/<stem>/`, named after the first file's
|
||||
stem.
|
||||
- **Growing an existing single file into a bundle.** `--page` extends an
|
||||
existing source page's `raw_files:`. If that raises the page from one file
|
||||
to more than one, the file it already had is folded into the new bundle
|
||||
alongside the ones just promoted, in the same call - at no point does
|
||||
`raw_files:` point at a path that does not exist.
|
||||
|
||||
Multi-owner raw files (`provenance.duplicate_raw_file_owners`) are refused
|
||||
rather than silently moved: relocating a file another page also claims would
|
||||
break that page's `raw_files:` without it ever being consulted.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
|
||||
from chemenu import config
|
||||
from chemenu.commands._util import fail, rel_path, success
|
||||
from chemenu.commands.dist_cmd import RAW_SUBDIRS
|
||||
from chemenu.frontmatter_io import write_page
|
||||
from chemenu.kb_scan import load_kb_pages
|
||||
from chemenu.provenance import source_pages_by_raw_file, source_raw_files
|
||||
|
||||
app = typer.Typer(help="Promote raw material out of incoming/ into raw/.")
|
||||
|
||||
|
||||
def _incoming_dir() -> Path:
|
||||
return config.ROOT / "incoming"
|
||||
|
||||
|
||||
def _resolve(raw: Path) -> Path:
|
||||
return raw if raw.is_absolute() else config.ROOT / raw
|
||||
|
||||
|
||||
def _classify(path: Path, incoming: Path) -> str:
|
||||
"""The type subdirectory `path` (already resolved, absolute) declares by
|
||||
where it sits under `incoming/`, or fail with the reason it doesn't."""
|
||||
try:
|
||||
rel = path.relative_to(incoming)
|
||||
except ValueError:
|
||||
fail(
|
||||
f"{rel_path(path)} is not under incoming/ - `raw accept` only promotes files "
|
||||
"from there. See raw/CONTRACT.md."
|
||||
)
|
||||
allowed = ", ".join(f"incoming/{s}/" for s in RAW_SUBDIRS)
|
||||
if len(rel.parts) < 2:
|
||||
fail(
|
||||
f"incoming/{rel} declares no type - place it inside one of {allowed} instead "
|
||||
"of directly in incoming/."
|
||||
)
|
||||
sub = rel.parts[0]
|
||||
if sub not in RAW_SUBDIRS:
|
||||
fail(f"incoming/{rel} lies under an unknown type directory 'incoming/{sub}/'. Allowed: {allowed}.")
|
||||
if len(rel.parts) > 2:
|
||||
fail(f"incoming/{rel} is nested below its type directory - place it directly in incoming/{sub}/.")
|
||||
return sub
|
||||
|
||||
|
||||
@app.command("accept")
|
||||
def raw_accept_command(
|
||||
files: list[Path] = typer.Argument(
|
||||
...,
|
||||
help="One or more files under incoming/<type>/, all belonging to the same source",
|
||||
),
|
||||
page: Optional[str] = typer.Option(
|
||||
None,
|
||||
"--page",
|
||||
help="Extend this existing source page's raw_files: with the promoted file(s), "
|
||||
"folding in its already-promoted file if this raises it past one",
|
||||
),
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="List what would move without writing"),
|
||||
):
|
||||
"""Promote file(s) from incoming/ into raw/, computing the destination
|
||||
(type directory, bundle or not, bundle name) instead of taking it as an
|
||||
argument. See raw/CONTRACT.md "Getting a file in: incoming/"."""
|
||||
if not files:
|
||||
fail("Pass at least one file to promote.")
|
||||
|
||||
incoming = _incoming_dir()
|
||||
resolved = [_resolve(f) for f in files]
|
||||
|
||||
for path in resolved:
|
||||
if not path.is_file():
|
||||
fail(f"{rel_path(path)} does not exist or is not a file.")
|
||||
|
||||
subs = {_classify(path, incoming) for path in resolved}
|
||||
if len(subs) > 1:
|
||||
allowed = ", ".join(sorted(f"incoming/{s}/" for s in subs))
|
||||
fail(f"All files in one `raw accept` call must share one type directory; got {allowed}.")
|
||||
sub = subs.pop()
|
||||
|
||||
names = [path.name for path in resolved]
|
||||
if len(names) != len(set(names)):
|
||||
fail("Two files share a filename; rename one before promoting.")
|
||||
|
||||
raw_sub_dir = config.RAW_DIR / sub
|
||||
|
||||
pages = None
|
||||
target_page = None
|
||||
existing_raw_paths: list[Path] = []
|
||||
if page is not None:
|
||||
pages = load_kb_pages(config.KB_DIR)
|
||||
target_page = pages.get(page)
|
||||
if target_page is None:
|
||||
fail(f"No page titled '{page}' found under wiki/. Create it first, or omit --page.")
|
||||
existing_rel = source_raw_files(target_page)
|
||||
if not existing_rel:
|
||||
fail(
|
||||
f"'{page}' has no raw_files: yet - omit --page and run "
|
||||
"`wikitool new source --set raw_files=...` for a page's first raw file."
|
||||
)
|
||||
existing_raw_paths = [config.ROOT / p for p in existing_rel]
|
||||
missing = [p for p in existing_raw_paths if not p.is_file()]
|
||||
if missing:
|
||||
fail(
|
||||
f"'{page}' claims raw file(s) that do not exist on disk: "
|
||||
f"{', '.join(rel_path(p) for p in missing)}. Fix raw_files: (see `sources coverage`) "
|
||||
"before promoting more."
|
||||
)
|
||||
existing_subs = {p.relative_to(config.RAW_DIR).parts[0] for p in existing_raw_paths}
|
||||
if existing_subs != {sub}:
|
||||
fail(
|
||||
f"'{page}' already claims file(s) under {', '.join(sorted(f'raw/{s}/' for s in existing_subs))}, "
|
||||
f"not raw/{sub}/. A bundle is one type directory; promote separately."
|
||||
)
|
||||
|
||||
# A bundle directory forms once two or more files belong to the source
|
||||
# (Gitea #58 decision 3): from the second file on, never before. Whenever
|
||||
# --page targets an existing page it always has >=1 raw file already
|
||||
# (types/source.md requires raw_files:), so bundling always applies there.
|
||||
total = len(existing_raw_paths) + len(resolved)
|
||||
bundle_dir: Optional[Path] = None
|
||||
if len(existing_raw_paths) >= 2:
|
||||
parents = {p.parent for p in existing_raw_paths}
|
||||
if len(parents) != 1:
|
||||
fail(
|
||||
f"'{page}' raw_files: are not all in one directory - fix them by hand first "
|
||||
"(see `sources coverage`)."
|
||||
)
|
||||
bundle_dir = parents.pop()
|
||||
elif total >= 2:
|
||||
primary = existing_raw_paths[0] if existing_raw_paths else resolved[0]
|
||||
bundle_dir = raw_sub_dir / primary.stem
|
||||
|
||||
moves: list[tuple[Path, Path]] = []
|
||||
for existing in existing_raw_paths:
|
||||
if bundle_dir is not None and existing.parent != bundle_dir:
|
||||
moves.append((existing, bundle_dir / existing.name))
|
||||
for new_path in resolved:
|
||||
dst = (bundle_dir / new_path.name) if bundle_dir is not None else (raw_sub_dir / new_path.name)
|
||||
moves.append((new_path, dst))
|
||||
|
||||
for _src, dst in moves:
|
||||
if dst.exists():
|
||||
fail(f"Cannot promote: {rel_path(dst)} already exists.")
|
||||
|
||||
moving_existing = [src for src, _dst in moves if src in existing_raw_paths]
|
||||
if moving_existing:
|
||||
by_raw = source_pages_by_raw_file(pages)
|
||||
conflicts = [
|
||||
(rel_path(src), sorted(set(by_raw.get(rel_path(src), [])) - {page}))
|
||||
for src in moving_existing
|
||||
]
|
||||
conflicts = [(p, owners) for p, owners in conflicts if owners]
|
||||
if conflicts:
|
||||
listed = "\n".join(f" - {p}: also claimed by {', '.join(o)}" for p, o in conflicts)
|
||||
fail(
|
||||
"Cannot bundle: the following already-covered raw file(s) have more than one "
|
||||
f"owner, so moving them would break the other page(s)' raw_files::\n{listed}\n"
|
||||
"Resolve the multiple ownership first (see `wikitool sources coverage`)."
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
for src, dst in moves:
|
||||
typer.echo(f"[dry-run] would move {rel_path(src)} -> {rel_path(dst)}")
|
||||
if target_page is not None:
|
||||
moved_map = dict(moves)
|
||||
final = [moved_map.get(p, p) for p in existing_raw_paths] + [moved_map[p] for p in resolved]
|
||||
typer.echo(f"[dry-run] would set raw_files: on '{page}' to {[rel_path(p) for p in final]}")
|
||||
typer.echo(f"[dry-run] would move {len(moves)} file(s). No files written.")
|
||||
return
|
||||
|
||||
for src, dst in moves:
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
src.rename(dst)
|
||||
typer.echo(f" moved {rel_path(src)} -> {rel_path(dst)}")
|
||||
|
||||
moved_map = dict(moves)
|
||||
if target_page is not None:
|
||||
final = [moved_map.get(p, p) for p in existing_raw_paths] + [moved_map[p] for p in resolved]
|
||||
target_page.frontmatter["raw_files"] = [rel_path(p) for p in final]
|
||||
write_page(target_page.path, target_page.frontmatter, target_page.body)
|
||||
success(
|
||||
f"Promoted {len(resolved)} file(s); updated raw_files: on '{page}' "
|
||||
f"({len(final)} file(s) total)."
|
||||
)
|
||||
return
|
||||
|
||||
promoted = ", ".join(rel_path(moved_map[p]) for p in resolved)
|
||||
success(
|
||||
f"Promoted {len(resolved)} file(s) to {promoted}. "
|
||||
"Run `wikitool new source --set raw_files=...` (or `touch --set` on an existing page) next."
|
||||
)
|
||||
@@ -371,6 +371,14 @@ def test_plan_creates_empty_raw_subdirs_not_real_content(repo):
|
||||
assert not any("personal-note" in relative for relative in plan)
|
||||
|
||||
|
||||
def test_plan_creates_matching_incoming_subdirs(repo):
|
||||
"""The ingest inbox (Gitea #58) mirrors raw/'s type subdirectories one for
|
||||
one - both come from the same `RAW_SUBDIRS` tuple."""
|
||||
plan = dist_cmd.build_plan()
|
||||
for sub in ("articles", "documents", "notes", "assets"):
|
||||
assert f"incoming/{sub}/.gitkeep" in plan
|
||||
|
||||
|
||||
def test_plan_seeds_log_and_changes_from_templates(repo):
|
||||
plan = dist_cmd.build_plan()
|
||||
assert "Wiki Log" in plan["kb/log.md"].content
|
||||
@@ -489,6 +497,7 @@ def test_export_into_a_fresh_directory_works(repo, tmp_path):
|
||||
assert (target / "AGENTS.md").is_file()
|
||||
assert (target / "kb" / "entities" / "COLLECTION.md.template").is_file()
|
||||
assert (target / "raw" / "notes" / ".gitkeep").is_file()
|
||||
assert (target / "incoming" / "notes" / ".gitkeep").is_file()
|
||||
|
||||
|
||||
def test_unbalanced_markers_fail_loudly(repo):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import pytest
|
||||
import typer
|
||||
|
||||
from chemenu import config
|
||||
from chemenu.commands import docs_verify
|
||||
|
||||
|
||||
@@ -133,6 +134,31 @@ def test_legacy_type_regex_matches_pre_migration_form():
|
||||
assert not docs_verify.LEGACY_TYPE_RE.search("---\ntype: types/comparison.md\n---")
|
||||
|
||||
|
||||
def test_this_repos_raw_subdirs_are_documented():
|
||||
assert docs_verify.check_raw_subdirs() == []
|
||||
|
||||
|
||||
def test_raw_subdirs_mismatch_is_reported(tmp_path, monkeypatch):
|
||||
"""The regression this guards: `dist_cmd.RAW_SUBDIRS` and raw/CONTRACT.md's
|
||||
routing table are two places naming the same set (AGENTS.md invariant 8),
|
||||
so either one drifting from the other must be caught in both directions."""
|
||||
(tmp_path / "raw").mkdir()
|
||||
(tmp_path / "raw" / "CONTRACT.md").write_text(
|
||||
"| Directory | Holds |\n|---|---|\n| `articles/` | ... |\n| `videos/` | ... |\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(config, "ROOT", tmp_path)
|
||||
monkeypatch.setattr(docs_verify.dist_cmd, "RAW_SUBDIRS", ("articles", "documents"))
|
||||
issues = docs_verify.check_raw_subdirs()
|
||||
assert any("missing `documents/`" in i for i in issues)
|
||||
assert any("lists `videos/`" in i for i in issues)
|
||||
|
||||
|
||||
def test_raw_subdirs_check_is_skipped_without_a_contract(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(config, "ROOT", tmp_path)
|
||||
assert docs_verify.check_raw_subdirs() == []
|
||||
|
||||
|
||||
def test_no_content_is_gitignored():
|
||||
"""The regression guard for the 2026-08-13 `.gitignore` rewrite: patterns
|
||||
like `*temp*` and `bin/` were silently excluding files under raw/, so the
|
||||
@@ -151,6 +177,14 @@ def test_a_swallowed_canary_is_reported():
|
||||
assert swallowed == ["tools/.wikitool_session/budget.json"]
|
||||
|
||||
|
||||
def test_incoming_inbox_is_ignored():
|
||||
"""Unlike raw/, a file under incoming/ must never be committed - promotion
|
||||
via `wikitool raw accept` is what makes it immutable, not the drop."""
|
||||
assert docs_verify.ignored_canaries(("incoming/documents/probe.pdf",)) == [
|
||||
"incoming/documents/probe.pdf"
|
||||
]
|
||||
|
||||
|
||||
def test_the_environment_note_is_ignored_but_its_template_is_not():
|
||||
"""The pattern has to split a file from its own template. `ENVIRONMENT.md`
|
||||
describes one checkout and must never be committed; `ENVIRONMENT.md.template`
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
import pytest
|
||||
import typer
|
||||
|
||||
from chemenu import config
|
||||
from chemenu.commands.raw_cmd import raw_accept_command
|
||||
from chemenu.frontmatter_io import read_page, write_page
|
||||
from chemenu.provenance import uncovered_raw_files
|
||||
from chemenu.kb_scan import load_kb_pages
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tree(kb_dir):
|
||||
"""kb_dir already repoints config.ROOT at tmp_path; add raw/ and
|
||||
incoming/ beside it, each with the four type subdirectories
|
||||
dist_cmd.RAW_SUBDIRS declares."""
|
||||
root = kb_dir.parent
|
||||
for sub in ("articles", "documents", "notes", "assets"):
|
||||
(root / "raw" / sub).mkdir(parents=True)
|
||||
(root / "incoming" / sub).mkdir(parents=True)
|
||||
return root
|
||||
|
||||
|
||||
def _accept(*files, page=None, dry_run=False):
|
||||
return raw_accept_command(files=list(files), page=page, dry_run=dry_run)
|
||||
|
||||
|
||||
def _write_source(kb_dir, title, raw_files):
|
||||
write_page(
|
||||
kb_dir / "sources" / f"{title}.md",
|
||||
{
|
||||
"type": "types/source.md", "source_type": "document", "author": "Torben",
|
||||
"raw_files": list(raw_files), "date": "2026-09-01",
|
||||
"tags": [], "entities": [], "concepts": [], "summary": "Test source.",
|
||||
},
|
||||
f"\n# {title}\n\n## Summary\n\nTest.\n",
|
||||
)
|
||||
|
||||
|
||||
def test_single_file_needs_no_bundle(tree):
|
||||
src = tree / "incoming/documents/handbuch.pdf"
|
||||
src.write_bytes(b"%PDF-1.4 fake\n")
|
||||
_accept(src)
|
||||
dst = tree / "raw/documents/handbuch.pdf"
|
||||
assert dst.is_file()
|
||||
assert dst.read_bytes() == b"%PDF-1.4 fake\n"
|
||||
assert not src.exists()
|
||||
|
||||
|
||||
def test_two_files_bundle_under_the_first_files_stem(tree):
|
||||
pdf = tree / "incoming/documents/handbuch.pdf"
|
||||
md = tree / "incoming/documents/handbuch.md"
|
||||
pdf.write_bytes(b"pdf-bytes")
|
||||
md.write_text("# converted\n", encoding="utf-8")
|
||||
_accept(pdf, md)
|
||||
assert (tree / "raw/documents/handbuch/handbuch.pdf").read_bytes() == b"pdf-bytes"
|
||||
assert (tree / "raw/documents/handbuch/handbuch.md").read_text(encoding="utf-8") == "# converted\n"
|
||||
assert not pdf.exists() and not md.exists()
|
||||
|
||||
|
||||
def test_file_directly_in_incoming_is_rejected(tree):
|
||||
src = tree / "incoming/stray.md"
|
||||
src.write_text("x\n", encoding="utf-8")
|
||||
with pytest.raises(typer.Exit):
|
||||
_accept(src)
|
||||
assert src.exists()
|
||||
|
||||
|
||||
def test_unknown_type_subdir_is_rejected(tree):
|
||||
(tree / "incoming/videos").mkdir()
|
||||
src = tree / "incoming/videos/clip.mp4"
|
||||
src.write_bytes(b"x")
|
||||
with pytest.raises(typer.Exit):
|
||||
_accept(src)
|
||||
assert src.exists()
|
||||
|
||||
|
||||
def test_nested_too_deep_is_rejected(tree):
|
||||
nested = tree / "incoming/documents/sub"
|
||||
nested.mkdir()
|
||||
src = nested / "deep.pdf"
|
||||
src.write_bytes(b"x")
|
||||
with pytest.raises(typer.Exit):
|
||||
_accept(src)
|
||||
assert src.exists()
|
||||
|
||||
|
||||
def test_mixed_type_subdirs_in_one_call_is_rejected(tree):
|
||||
a = tree / "incoming/documents/a.pdf"
|
||||
b = tree / "incoming/notes/b.md"
|
||||
a.write_bytes(b"a")
|
||||
b.write_text("b", encoding="utf-8")
|
||||
with pytest.raises(typer.Exit):
|
||||
_accept(a, b)
|
||||
assert a.exists() and b.exists()
|
||||
|
||||
|
||||
def test_same_file_passed_twice_is_rejected(tree):
|
||||
src = tree / "incoming/documents/handbuch.pdf"
|
||||
src.write_bytes(b"x")
|
||||
with pytest.raises(typer.Exit):
|
||||
_accept(src, src)
|
||||
assert src.exists()
|
||||
|
||||
|
||||
def test_collision_with_existing_raw_file_is_rejected(tree):
|
||||
(tree / "raw/documents/handbuch.pdf").write_bytes(b"already there")
|
||||
src = tree / "incoming/documents/handbuch.pdf"
|
||||
src.write_bytes(b"new")
|
||||
with pytest.raises(typer.Exit):
|
||||
_accept(src)
|
||||
assert src.exists()
|
||||
assert (tree / "raw/documents/handbuch.pdf").read_bytes() == b"already there"
|
||||
|
||||
|
||||
def test_missing_file_is_rejected(tree):
|
||||
with pytest.raises(typer.Exit):
|
||||
_accept(tree / "incoming/documents/absent.pdf")
|
||||
|
||||
|
||||
def test_dry_run_moves_nothing(tree):
|
||||
src = tree / "incoming/documents/handbuch.pdf"
|
||||
src.write_bytes(b"x")
|
||||
_accept(src, dry_run=True)
|
||||
assert src.exists()
|
||||
assert not (tree / "raw/documents/handbuch.pdf").exists()
|
||||
|
||||
|
||||
def test_promoted_file_in_incoming_is_never_reported_uncovered(tree):
|
||||
"""AC: incoming/ is invisible to sources coverage / lint until accepted -
|
||||
both walk raw/ only."""
|
||||
(tree / "incoming/notes/not-yet-promoted.md").write_text("draft\n", encoding="utf-8")
|
||||
pages = load_kb_pages(tree / "kb")
|
||||
assert "raw/notes/not-yet-promoted.md" not in uncovered_raw_files(tree / "raw", pages)
|
||||
assert uncovered_raw_files(tree / "raw", pages) == []
|
||||
|
||||
|
||||
def test_without_page_flag_only_moves_and_prints_target(tree, capsys):
|
||||
src = tree / "incoming/notes/meeting.md"
|
||||
src.write_text("notes\n", encoding="utf-8")
|
||||
_accept(src)
|
||||
out = capsys.readouterr().out
|
||||
assert "raw/notes/meeting.md" in out
|
||||
assert "new source" in out
|
||||
|
||||
|
||||
def test_page_flag_extends_raw_files_for_a_single_new_file(tree):
|
||||
"""No growth case: the page already has >=1 file, so this always bundles -
|
||||
see test_growth_case below for the interesting path."""
|
||||
(tree / "raw/documents").mkdir(exist_ok=True)
|
||||
_write_source(tree / "kb", "Source - Handbuch", ["raw/documents/handbuch.pdf"])
|
||||
(tree / "raw/documents/handbuch.pdf").write_bytes(b"first")
|
||||
new_file = tree / "incoming/documents/handbuch-appendix.pdf"
|
||||
new_file.write_bytes(b"second")
|
||||
|
||||
_accept(new_file, page="Source - Handbuch")
|
||||
|
||||
bundle = tree / "raw/documents/handbuch"
|
||||
assert (bundle / "handbuch.pdf").read_bytes() == b"first"
|
||||
assert (bundle / "handbuch-appendix.pdf").read_bytes() == b"second"
|
||||
assert not (tree / "raw/documents/handbuch.pdf").exists()
|
||||
|
||||
frontmatter, _ = read_page(tree / "kb/sources/Source - Handbuch.md")
|
||||
assert sorted(frontmatter["raw_files"]) == sorted(
|
||||
["raw/documents/handbuch/handbuch.pdf", "raw/documents/handbuch/handbuch-appendix.pdf"]
|
||||
)
|
||||
|
||||
|
||||
def test_growth_case_no_broken_or_uncovered_refs_afterwards(tree):
|
||||
_write_source(tree / "kb", "Source - Handbuch", ["raw/documents/handbuch.pdf"])
|
||||
(tree / "raw/documents/handbuch.pdf").write_bytes(b"first")
|
||||
new_file = tree / "incoming/documents/handbuch.md"
|
||||
new_file.write_text("converted", encoding="utf-8")
|
||||
|
||||
_accept(new_file, page="Source - Handbuch")
|
||||
|
||||
from chemenu.provenance import broken_raw_refs
|
||||
|
||||
pages = load_kb_pages(tree / "kb")
|
||||
# kb_dir ships an unrelated "Source - Aurora" fixture page whose raw file
|
||||
# this tree never had - filter to what this test actually changed.
|
||||
assert [r for r in broken_raw_refs(pages) if r["page"] == "Source - Handbuch"] == []
|
||||
assert [f for f in uncovered_raw_files(tree / "raw", pages) if "handbuch" in f] == []
|
||||
|
||||
|
||||
def test_adding_to_an_already_bundled_source_joins_the_existing_bundle(tree):
|
||||
(tree / "raw/documents/handbuch").mkdir(parents=True)
|
||||
(tree / "raw/documents/handbuch/handbuch.pdf").write_bytes(b"a")
|
||||
(tree / "raw/documents/handbuch/handbuch.md").write_text("b", encoding="utf-8")
|
||||
_write_source(
|
||||
tree / "kb", "Source - Handbuch",
|
||||
["raw/documents/handbuch/handbuch.pdf", "raw/documents/handbuch/handbuch.md"],
|
||||
)
|
||||
extra = tree / "incoming/documents/handbuch-notes.md"
|
||||
extra.write_text("c", encoding="utf-8")
|
||||
|
||||
_accept(extra, page="Source - Handbuch")
|
||||
|
||||
assert (tree / "raw/documents/handbuch/handbuch-notes.md").read_text(encoding="utf-8") == "c"
|
||||
# Nothing already-bundled was moved a second time.
|
||||
assert (tree / "raw/documents/handbuch/handbuch.pdf").read_bytes() == b"a"
|
||||
frontmatter, _ = read_page(tree / "kb/sources/Source - Handbuch.md")
|
||||
assert set(frontmatter["raw_files"]) == {
|
||||
"raw/documents/handbuch/handbuch.pdf",
|
||||
"raw/documents/handbuch/handbuch.md",
|
||||
"raw/documents/handbuch/handbuch-notes.md",
|
||||
}
|
||||
|
||||
|
||||
def test_growth_case_rejects_a_multi_owner_raw_file(tree):
|
||||
(tree / "raw/documents/handbuch.pdf").write_bytes(b"shared")
|
||||
_write_source(tree / "kb", "Source - Handbuch", ["raw/documents/handbuch.pdf"])
|
||||
_write_source(tree / "kb", "Source - Also Handbuch", ["raw/documents/handbuch.pdf"])
|
||||
new_file = tree / "incoming/documents/handbuch.md"
|
||||
new_file.write_text("x", encoding="utf-8")
|
||||
|
||||
with pytest.raises(typer.Exit):
|
||||
_accept(new_file, page="Source - Handbuch")
|
||||
|
||||
assert new_file.exists()
|
||||
assert (tree / "raw/documents/handbuch.pdf").exists()
|
||||
assert not (tree / "raw/documents/handbuch").exists()
|
||||
|
||||
|
||||
def test_page_not_found_is_rejected(tree):
|
||||
new_file = tree / "incoming/documents/handbuch.md"
|
||||
new_file.write_text("x", encoding="utf-8")
|
||||
with pytest.raises(typer.Exit):
|
||||
_accept(new_file, page="Source - Nonexistent")
|
||||
assert new_file.exists()
|
||||
|
||||
|
||||
def test_page_with_no_raw_files_is_rejected(tree):
|
||||
write_page(
|
||||
tree / "kb/sources/Source - Empty.md",
|
||||
{
|
||||
"type": "types/source.md", "source_type": "document", "author": "Torben",
|
||||
"raw_files": [], "date": "2026-09-01",
|
||||
"tags": [], "entities": [], "concepts": [], "summary": "Empty.",
|
||||
},
|
||||
"\n# Source - Empty\n",
|
||||
)
|
||||
new_file = tree / "incoming/documents/handbuch.md"
|
||||
new_file.write_text("x", encoding="utf-8")
|
||||
with pytest.raises(typer.Exit):
|
||||
_accept(new_file, page="Source - Empty")
|
||||
assert new_file.exists()
|
||||
|
||||
|
||||
def test_type_directory_mismatch_with_existing_raw_files_is_rejected(tree):
|
||||
(tree / "raw/notes/handbuch.md").write_bytes(b"first")
|
||||
_write_source(tree / "kb", "Source - Handbuch", ["raw/notes/handbuch.md"])
|
||||
new_file = tree / "incoming/documents/handbuch.pdf"
|
||||
new_file.write_bytes(b"second")
|
||||
with pytest.raises(typer.Exit):
|
||||
_accept(new_file, page="Source - Handbuch")
|
||||
assert new_file.exists()
|
||||
Reference in New Issue
Block a user