feat: incoming/ as a queue - raw pending picks the next entry, raw accept takes a whole folder, a file in a subdirectory of incoming/ is refused (#112)

Files changed:
- CHANGES.md
- README.md
- VERSION
- instructions/ingest-large-tree.md
- instructions/wiki-ingest/SKILL.md
- raw/CONTRACT.md
- tools/CONTRACT.md
- tools/chemenu/cli_contract.py
- tools/chemenu/commands/raw_cmd.py
- tools/chemenu/tests/test_raw_cmd.py
- tools/chemenu/tests/test_raw_fetch.py

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2

tools/chemenu/tests/test_raw_cmd.py

@@ -1,10 +1,13 @@
 import datetime
+import json
+import os
+import re
 
 import pytest
 import typer
 
 from chemenu import config
-from chemenu.commands.raw_cmd import raw_accept_command
+from chemenu.commands.raw_cmd import raw_accept_command, raw_pending_command
 from chemenu.frontmatter_io import read_page, write_page
 from chemenu.provenance import uncovered_raw_files
 from chemenu.kb_scan import load_kb_pages
@@ -85,40 +88,21 @@ def test_file_directly_in_incoming_is_accepted(tree):
     assert not src.exists()
 
 
-def test_subdirectory_under_incoming_is_ignored_not_inspected(tree):
-    """Gitea #67: a subdirectory of incoming/ - old habit, old script - is
-    tolerated and ignored rather than read as a type classification. This is
-    the MINOR condition named in the issue's Versionsteil."""
-    (tree / "incoming/videos").mkdir()
-    src = tree / "incoming/videos/clip.mp4"
-    src.write_bytes(b"x")
-    _accept(src)
-    assert (tree / _shard() / "clip.mp4").is_file()
-    assert not src.exists()
-
-
-def test_nested_too_deep_is_rejected(tree):
-    nested = tree / "incoming/documents/sub"
-    nested.mkdir(parents=True)
-    src = nested / "deep.pdf"
+@pytest.mark.parametrize("rel", ["documents/a.pdf", "documents/sub/a.pdf"])
+def test_file_in_a_subdirectory_is_refused_naming_both_routes(tree, capsys, rel):
+    """Gitea #112: a subdirectory of incoming/ is a source of its own, no
+    longer a tolerated, ignored type directory - a file inside one is refused,
+    and the message names accepting the folder and moving the file up."""
+    src = tree / "incoming" / rel
+    src.parent.mkdir(parents=True)
     src.write_bytes(b"x")
+    before = _tree_files(tree)
     with pytest.raises(typer.Exit):
         _accept(src)
-    assert src.exists()
-
-
-def test_files_from_different_ignored_subdirs_bundle_together(tree):
-    """No more per-call type agreement to enforce (Gitea #67): which ignored
-    subdirectory each file happened to sit under is irrelevant now."""
-    a = tree / "incoming/documents/a.pdf"
-    b = tree / "incoming/notes/a.md"
-    a.parent.mkdir(parents=True)
-    b.parent.mkdir(parents=True)
-    a.write_bytes(b"a")
-    b.write_text("b", encoding="utf-8")
-    _accept(a, b)
-    bundle = tree / _shard() / "a"
-    assert (bundle / "a.pdf").exists() and (bundle / "a.md").exists()
+    assert _tree_files(tree) == before
+    out = " ".join(capsys.readouterr().out.split())
+    assert "raw accept incoming/documents" in out
+    assert "move the file up into incoming/" in out
 
 
 def test_same_file_passed_twice_is_rejected(tree):
@@ -573,15 +557,13 @@ def test_replaces_rejects_filename_mismatch(tree):
     assert new.exists()
 
 
-def test_replaces_across_legacy_directories_is_now_allowed(tree):
-    """Gitea #67 removes the type-directory-match check `--replaces` used to
-    enforce: a subdirectory of incoming/ carries no meaning any more, so
-    replacing a raw/notes/ file with an incoming file dropped under an
-    unrelated incoming/documents/ works exactly like one dropped flat."""
+def test_replaces_writes_back_into_a_legacy_directory(tree):
+    """Gitea #67 removed the type-directory-match check `--replaces` used to
+    enforce: a raw/notes/ file is replaced by a file dropped flat into
+    incoming/ like any other."""
     target = tree / "raw/notes/handbuch.md"
     target.write_text("old", encoding="utf-8")
-    new = tree / "incoming/documents/handbuch.md"
-    new.parent.mkdir(parents=True)
+    new = tree / "incoming/handbuch.md"
     new.write_text("new", encoding="utf-8")
 
     _accept(new, replaces=target, fidelity=None, authority=None)
@@ -590,6 +572,21 @@ def test_replaces_across_legacy_directories_is_now_allowed(tree):
     assert not new.exists()
 
 
+def test_replaces_refuses_an_incoming_file_in_a_subdirectory(tree):
+    """Gitea #112: the subdirectory tolerance is gone for --replaces too."""
+    target = tree / "raw/notes/handbuch.md"
+    target.write_text("old", encoding="utf-8")
+    new = tree / "incoming/documents/handbuch.md"
+    new.parent.mkdir(parents=True)
+    new.write_text("new", encoding="utf-8")
+
+    with pytest.raises(typer.Exit):
+        _accept(new, replaces=target, fidelity=None, authority=None)
+
+    assert target.read_text(encoding="utf-8") == "old"
+    assert new.read_text(encoding="utf-8") == "new"
+
+
 def test_replaces_rejects_nonexistent_target(tree):
     new = tree / "incoming/handbuch.md"
     new.write_text("new", encoding="utf-8")
@@ -654,3 +651,263 @@ def test_accept_takes_a_target_at_the_path_budget(tree):
     _accept(src)
     assert (tree / _shard() / src.name).exists()
     assert not src.exists()
+
+
+# --- A folder is one source (Gitea #112) ---
+
+
+def _make_folder(tree, name, files):
+    folder = tree / "incoming" / name
+    for rel, data in files.items():
+        (folder / rel).parent.mkdir(parents=True, exist_ok=True)
+        (folder / rel).write_bytes(data)
+    return folder
+
+
+_BAUM = {"a.md": b"A", "sub/b.md": b"B", "sub/deep/README.md": b"R"}
+
+
+def test_folder_moves_every_file_byte_identical_and_leaves_no_folder(tree):
+    folder = _make_folder(tree, "baum", _BAUM)
+    _accept(folder)
+    bundle = tree / _shard() / "baum"
+    assert {str(p.relative_to(bundle)): p.read_bytes() for p in bundle.rglob("*") if p.is_file()} == _BAUM
+    assert not folder.exists()
+    assert sorted(p.name for p in (tree / "incoming").iterdir()) == []
+
+
+def test_folder_takes_same_named_files_in_different_subfolders(tree):
+    folder = _make_folder(tree, "doku", {"x/README.md": b"x", "y/README.md": b"y"})
+    _accept(folder)
+    bundle = tree / _shard() / "doku"
+    assert (bundle / "x/README.md").read_bytes() == b"x"
+    assert (bundle / "y/README.md").read_bytes() == b"y"
+
+
+def test_folder_prints_raw_files_and_capture_pair(tree, capsys):
+    folder = _make_folder(tree, "baum", _BAUM)
+    _accept(folder, fidelity="secondhand", authority="opinion")
+    out = "".join(capsys.readouterr().out.split())  # success() wraps long lines
+    assert f"--setraw_files={_shard()}/baum/a.md,{_shard()}/baum/sub/b.md,{_shard()}/baum/sub/deep/README.md" in out
+    assert "--setfidelity=secondhand--setauthority=opinion" in out
+
+
+@pytest.mark.parametrize("extra", ["sub/.hidden", ".git/config", "link"])
+def test_folder_with_hidden_entry_or_symlink_is_refused_and_named(tree, capsys, extra):
+    folder = _make_folder(tree, "baum", _BAUM)
+    if extra == "link":
+        (folder / "sub/link").symlink_to(folder / "a.md")
+        named = "baum/sub/link"
+    else:
+        (folder / extra).parent.mkdir(parents=True, exist_ok=True)
+        (folder / extra).write_bytes(b"h")
+        named = f"baum/{extra.split('/')[0]}" if extra.startswith(".") else f"baum/{extra}"
+    before = _tree_files(tree)
+    links_before = sorted(str(p) for p in tree.rglob("*") if p.is_symlink())
+    with pytest.raises(typer.Exit):
+        _accept(folder)
+    assert _tree_files(tree) == before
+    assert sorted(str(p) for p in tree.rglob("*") if p.is_symlink()) == links_before
+    assert named in capsys.readouterr().out
+
+
+def test_empty_folder_is_refused(tree):
+    (tree / "incoming/leer/sub").mkdir(parents=True)
+    with pytest.raises(typer.Exit):
+        _accept(tree / "incoming/leer")
+    assert (tree / "incoming/leer/sub").is_dir()
+
+
+@pytest.mark.parametrize("combination", ["other-file", "page", "replaces"])
+def test_folder_with_anything_else_is_refused(tree, combination):
+    folder = _make_folder(tree, "baum", _BAUM)
+    other = tree / "incoming/other.md"
+    other.write_text("o", encoding="utf-8")
+    (tree / "raw/notes/a.md").write_bytes(b"old")
+    _write_source(tree / "kb", "Source - Old", ["raw/notes/a.md"])
+    before = _tree_files(tree)
+    with pytest.raises(typer.Exit):
+        if combination == "other-file":
+            _accept(folder, other)
+        elif combination == "page":
+            _accept(folder, page="Source - Old")
+        else:
+            _accept(folder, replaces=tree / "raw/notes/a.md", fidelity=None, authority=None)
+    assert _tree_files(tree) == before
+
+
+def test_folder_needs_both_capture_fields(tree):
+    folder = _make_folder(tree, "baum", _BAUM)
+    with pytest.raises(typer.Exit):
+        raw_accept_command(files=[folder], fidelity=None, authority="reporting", page=None, replaces=None, dry_run=False)
+    assert folder.is_dir()
+
+
+def test_folder_name_already_occupied_is_refused_with_the_rename_route(tree, capsys):
+    (tree / "raw/documents/baum.pdf").write_bytes(b"pdf")
+    folder = _make_folder(tree, "baum", _BAUM)
+    before = _tree_files(tree)
+    with pytest.raises(typer.Exit):
+        _accept(folder)
+    assert _tree_files(tree) == before
+    out = " ".join(capsys.readouterr().out.split())
+    assert "Rename the folder in incoming/" in out
+    assert "--replaces raw/" not in out
+
+
+def test_folder_target_over_the_path_budget_is_refused_before_any_move(tree):
+    long_name = "n" * (161 - len(f"{_shard()}/baum/sub/") - len(".md")) + ".md"
+    folder = _make_folder(tree, "baum", {"a.md": b"A", f"sub/{long_name}": b"L"})
+    before = _tree_files(tree)
+    with pytest.raises(typer.Exit):
+        _accept(folder)
+    assert _tree_files(tree) == before
+
+
+def test_folder_dry_run_lists_every_move_and_the_folder_and_changes_nothing(tree, capsys):
+    folder = _make_folder(tree, "baum", _BAUM)
+    before = _tree_files(tree)
+    _accept(folder, dry_run=True)
+    assert _tree_files(tree) == before
+    out = capsys.readouterr().out
+    for rel in _BAUM:
+        assert f"incoming/baum/{rel} -> {_shard()}/baum/{rel}" in out
+    assert "would remove incoming/baum/" in out
+
+
+def test_folder_files_are_uncovered_not_broken_afterwards(tree):
+    folder = _make_folder(tree, "baum", _BAUM)
+    _accept(folder)
+    from chemenu.provenance import broken_raw_refs
+
+    pages = load_kb_pages(tree / "kb")
+    assert sorted(f for f in uncovered_raw_files(tree / "raw", pages) if "/baum/" in f) == sorted(
+        f"{_shard()}/baum/{rel}" for rel in _BAUM
+    )
+    assert [r for r in broken_raw_refs(pages) if "baum" in str(r)] == []
+
+
+def test_file_and_bundle_at_the_top_still_work(tree):
+    single = tree / "incoming/a.pdf"
+    single.write_bytes(b"a")
+    _accept(single)
+    assert (tree / _shard() / "a.pdf").read_bytes() == b"a"
+    html, md = tree / "incoming/post.html", tree / "incoming/post.md"
+    html.write_bytes(b"h")
+    md.write_bytes(b"m")
+    _accept(html, md)
+    assert sorted(p.name for p in (tree / _shard() / "post").iterdir()) == ["post.html", "post.md"]
+
+
+# --- raw pending (Gitea #112) ---
+
+
+def _pending(capsys):
+    capsys.readouterr()
+    raw_pending_command(json_out=True)
+    return json.loads(capsys.readouterr().out)
+
+
+def _age(path, seconds):
+    for p in [path, *path.rglob("*")] if path.is_dir() else [path]:
+        os.utime(p, ns=(seconds * 10**9, seconds * 10**9), follow_symlinks=False)
+
+
+def test_pending_on_an_empty_inbox_is_nothing(tree, capsys):
+    (tree / "incoming/.gitkeep").write_bytes(b"")
+    (tree / "incoming/documents").mkdir()
+    (tree / "incoming/notes/sub").mkdir(parents=True)
+    assert _pending(capsys) == []
+    raw_pending_command(json_out=False)
+    assert "Nothing is waiting" in capsys.readouterr().out
+
+
+def test_pending_groups_a_fetch_pair_into_one_bundle(tree, capsys):
+    (tree / "incoming/post.html").write_bytes(b"h")
+    (tree / "incoming/post.md").write_bytes(b"m")
+    [candidate] = _pending(capsys)
+    assert candidate["kind"] == "bundle"
+    assert candidate["paths"] == ["incoming/post.html", "incoming/post.md"]
+    assert candidate["files"] == 2
+
+
+def test_pending_counts_a_folder_once_with_all_its_files(tree, capsys):
+    _make_folder(tree, "baum", _BAUM)
+    [candidate] = _pending(capsys)
+    assert candidate["kind"] == "folder"
+    assert candidate["paths"] == ["incoming/baum"]
+    assert candidate["files"] == 3
+
+
+def test_pending_orders_oldest_first_by_the_newest_part_then_by_name(tree, capsys):
+    a = tree / "incoming/a.md"
+    a.write_bytes(b"a")
+    _age(a, 3000)
+    b_html, b_md = tree / "incoming/b.html", tree / "incoming/b.md"
+    b_html.write_bytes(b"h")
+    b_md.write_bytes(b"m")
+    _age(b_html, 1000)
+    _age(b_md, 4000)  # the bundle is as new as its newest part
+    folder = _make_folder(tree, "c", {"x.md": b"x", "y/z.md": b"z"})
+    _age(folder, 1000)
+    _age(folder / "y/z.md", 2000)
+    tie = tree / "incoming/0-tie.md"
+    tie.write_bytes(b"t")
+    _age(tie, 2000)
+    order = [c["paths"][0] for c in _pending(capsys)]
+    assert order == ["incoming/0-tie.md", "incoming/c", "incoming/a.md", "incoming/b.html"]
+
+
+def test_pending_marks_what_raw_accept_would_refuse_and_defaults_to_the_first_acceptable(tree, capsys):
+    hidden = _make_folder(tree, "hidden", {"a.md": b"a", ".DS_Store": b"x"})
+    _age(hidden, 1000)
+    (tree / "raw/documents/taken.pdf").write_bytes(b"pdf")
+    taken = tree / "incoming/taken.md"
+    taken.write_bytes(b"t")
+    _age(taken, 2000)
+    long = tree / "incoming" / _incoming_name_for(161)
+    long.write_bytes(b"l")
+    _age(long, 3000)
+    fine = tree / "incoming/fine.md"
+    fine.write_bytes(b"f")
+    _age(fine, 4000)
+
+    candidates = _pending(capsys)
+    assert [c["acceptable"] for c in candidates] == [False, False, False, True]
+    assert ".DS_Store" in candidates[0]["reason"]
+    assert "already claims" in candidates[1]["reason"]
+    assert "160" in candidates[2]["reason"] or "budget" in candidates[2]["reason"].lower()
+    assert [c["default"] for c in candidates] == [False, False, False, True]
+
+    raw_pending_command(json_out=False)
+    out = capsys.readouterr().out
+    assert "Default (*): incoming/fine.md" in out
+    assert "not acceptable" in out
+
+
+def test_pending_changes_nothing_not_even_mtimes(tree, capsys):
+    _make_folder(tree, "baum", _BAUM)
+    (tree / "incoming/post.html").write_bytes(b"h")
+    (tree / "raw/documents/old.pdf").write_bytes(b"o")
+
+    def snapshot():
+        return {
+            str(p.relative_to(tree)): (p.read_bytes() if p.is_file() else None, p.lstat().st_mtime_ns)
+            for root in ("incoming", "raw", "kb") for p in sorted((tree / root).rglob("*"))
+        }
+
+    before = snapshot()
+    _pending(capsys)
+    raw_pending_command(json_out=False)
+    assert snapshot() == before
+
+
+def test_pending_json_and_text_list_the_same_candidates_in_the_same_order(tree, capsys):
+    for name, age in (("a.md", 3000), ("b.md", 1000), ("c.md", 2000)):
+        (tree / "incoming" / name).write_bytes(b"x")
+        _age(tree / "incoming" / name, age)
+    paths = [c["paths"][0] for c in _pending(capsys)]
+    raw_pending_command(json_out=False)
+    text = capsys.readouterr().out
+    listed = [m.group(1) for m in re.finditer(r"^[* ] \d+\. \w+\s+(\S+)", text, re.MULTILINE)]
+    assert listed == paths

tools/chemenu/tests/test_raw_fetch.py

@@ -371,6 +371,18 @@ def test_html_outside_incoming_is_refused(tree):
     assert list((tree / "incoming").iterdir()) == []
 
 
+def test_html_in_a_subdirectory_of_incoming_is_refused(tree, capsys):
+    """A subdirectory of incoming/ is a source of its own (Gitea #112), so a
+    saved page inside one is not derived in place."""
+    saved = tree / "incoming/articles/post.html"
+    saved.parent.mkdir()
+    saved.write_bytes(ARTICLE)
+    before = _snapshot(tree)
+    _refused(html=saved, source_url="https://example.org/post")
+    assert _snapshot(tree) == before
+    assert "raw accept incoming/articles" in " ".join(capsys.readouterr().out.split())
+
+
 def test_html_without_url_is_refused(tree):
     saved = tree / "incoming/post.html"
     saved.write_bytes(ARTICLE)