chemenu/tools/chemenu/tests/test_collect.py

"""Tests for the pulled intake channels (`chemenu/collect.py`) and `wikitool
upload collect`: the `folder` and `command` handlers, and what the command does
across several channels."""
from __future__ import annotations

import json
import os
import subprocess
import sys
import time
from pathlib import Path

import pytest
import typer

from chemenu import collect, config, upload
from chemenu.errors import ValidationError
from chemenu.commands.upload_cmd import upload_collect_command, upload_list_command

OLD = time.time() - 3600


@pytest.fixture
def root(tmp_path, monkeypatch):
    instance = tmp_path / "instance"
    instance.mkdir()
    (instance / "incoming").mkdir()
    monkeypatch.setattr(config, "ROOT", instance)
    return instance


@pytest.fixture
def drop(tmp_path):
    folder = tmp_path / "drop"
    folder.mkdir()
    return folder


def _put(folder: Path, name: str, data: bytes = b"content\n", age: float = 3600) -> Path:
    path = folder / name
    path.write_bytes(data)
    stamp = time.time() - age
    os.utime(path, (stamp, stamp))
    return path


def _folder_channel(drop: Path, **overrides) -> upload.Channel:
    fields = dict(
        name="onedrive", handler="folder", max_bytes=1024, allowed_extensions=(".md", ".pdf"),
        path=drop, settle_seconds=60,
    )
    fields.update(overrides)
    return upload.Channel(**fields)


def _command_channel(argv: list[str], **overrides) -> upload.Channel:
    fields = dict(
        name="mail", handler="command", max_bytes=1024, allowed_extensions=(".md", ".pdf"),
        command_name="scan", command=tuple(argv),
    )
    fields.update(overrides)
    return upload.Channel(**fields)


def _script(body: str) -> list[str]:
    return [sys.executable, "-c", f"import sys, pathlib\nout = pathlib.Path(sys.argv[1])\n{body}"]


def _write_intake(root: Path, body: dict) -> None:
    (root / config.LOCAL_SETTINGS_FILENAME).write_text(
        json.dumps({"schema": 1, "intake": body}), encoding="utf-8")


def _tree(folder: Path) -> dict[str, bytes]:
    return {str(p.relative_to(folder)): p.read_bytes() for p in sorted(folder.rglob("*")) if p.is_file()}


def _manifests(root: Path) -> list[dict]:
    return upload.list_submissions(root)


# --- folder -----------------------------------------------------------------

def test_a_folder_pull_stores_each_file_and_empties_the_source(root, drop):
    _put(drop, "a.md", b"alpha\n")
    _put(drop, "b.pdf", b"bravo\n")
    result = collect.collect(root, _folder_channel(drop))
    assert result.ok and len(result.taken) == 2
    assert _tree(drop) == {}
    manifests = {m["filename"]: m for m in _manifests(root)}
    assert set(manifests) == {"a.md", "b.pdf"}
    first = manifests["a.md"]
    assert (first["channel"], first["handler"]) == ("onedrive", "folder")
    assert first["origin"] == str(drop / "a.md")
    assert (first["submitter"], first["submitter_source"]) == ("unknown", "folder")
    assert (root / "quarantine" / first["id"] / "a.md").read_bytes() == b"alpha\n"


def test_a_folder_pull_leaves_what_it_does_not_take_untouched(root, drop):
    (drop / "sub").mkdir()
    (drop / "sub" / "inner.md").write_bytes(b"inner")
    _put(drop, ".x.pdf", b"hidden")
    _put(drop, "~$a.docx", b"lock")
    _put(drop, "notes.exe", b"bad extension")
    _put(drop, "big.md", b"x" * 2000)
    target = _put(drop, "real.md", b"target")
    link = drop / "link.md"
    link.symlink_to(target)
    target.unlink()
    before = _tree(drop)
    result = collect.collect(root, _folder_channel(drop))
    assert result.ok and result.taken == [] and _manifests(root) == []
    assert _tree(drop) == before
    assert link.is_symlink()
    reasons = dict(result.skipped)
    assert reasons["sub"] == "subdirectory"
    assert reasons[".x.pdf"] == "hidden" and reasons["~$a.docx"] == "hidden"
    assert reasons["notes.exe"] == "extension not allowed"
    assert reasons["link.md"] == "symbolic link"
    assert "max_bytes" in reasons["big.md"]


def test_a_file_younger_than_settle_seconds_waits_for_the_next_run(root, drop):
    _put(drop, "fresh.md", b"arriving", age=5)
    result = collect.collect(root, _folder_channel(drop, settle_seconds=60))
    assert result.taken == [] and "still in motion" in dict(result.skipped)["fresh.md"]
    assert (drop / "fresh.md").read_bytes() == b"arriving"
    os.utime(drop / "fresh.md", (OLD, OLD))
    later = collect.collect(root, _folder_channel(drop, settle_seconds=60))
    assert [name for name, _ in later.taken] == ["fresh.md"]


def test_a_source_that_changes_during_the_copy_is_kept_and_nothing_is_stored(root, drop, monkeypatch):
    source = _put(drop, "a.md", b"one\n")
    monkeypatch.setattr(collect, "source_matches", lambda *args: False)
    result = collect.collect(root, _folder_channel(drop))
    assert result.taken == [] and "changed while it was copied" in dict(result.skipped)["a.md"]
    assert source.read_bytes() == b"one\n"
    assert _manifests(root) == []
    assert [p for p in (root / "quarantine").iterdir() if p.is_dir()] == []


def test_a_duplicate_is_reported_with_the_existing_id_and_the_source_is_removed(root, drop):
    _put(drop, "a.md", b"same\n")
    first = collect.collect(root, _folder_channel(drop))
    existing = first.taken[0][1]
    _put(drop, "again.md", b"same\n")
    second = collect.collect(root, _folder_channel(drop))
    assert second.taken == [] and second.duplicates == [("again.md", existing)]
    assert not (drop / "again.md").exists()
    assert len(_manifests(root)) == 1


def test_a_missing_folder_fails_the_channel(root, tmp_path):
    result = collect.collect(root, _folder_channel(tmp_path / "gone"))
    assert not result.ok and "does not exist" in result.error


# --- command ----------------------------------------------------------------

def test_a_command_hands_over_the_files_it_left_in_the_staging_directory(root):
    channel = _command_channel(_script("(out / 'mail.md').write_bytes(b'from a script\\n')"))
    result = collect.collect(root, channel)
    assert result.ok and [name for name, _ in result.taken] == ["mail.md"]
    manifest = _manifests(root)[0]
    assert (manifest["channel"], manifest["handler"], manifest["origin"]) == ("mail", "command", "command:scan")
    assert (manifest["submitter"], manifest["submitter_source"]) == ("unknown", "command")
    assert not (root / "quarantine" / ".staging").exists()


def test_a_failing_command_takes_nothing_and_shows_its_stderr_tail(root):
    channel = _command_channel(_script(
        "(out / 'half.md').write_bytes(b'x')\nsys.stderr.write('boom ' * 600 + 'THE END')\nsys.exit(3)"))
    result = collect.collect(root, channel)
    assert not result.ok and result.taken == []
    assert "exit 3" in result.error and result.error.rstrip().endswith("THE END")
    assert len(result.error) < 2500
    assert _manifests(root) == []
    assert not (root / "quarantine" / ".staging").exists()


def test_a_command_that_times_out_fails_its_channel_and_cleans_up(root, monkeypatch):
    monkeypatch.setattr(collect, "COMMAND_TIMEOUT_SECONDS", 1)
    channel = _command_channel(_script("(out / 'a.md').write_bytes(b'x')\nimport time\ntime.sleep(30)"))
    result = collect.collect(root, channel)
    assert not result.ok and "did not finish" in result.error
    assert _manifests(root) == []
    assert not (root / "quarantine" / ".staging").exists()


def test_a_command_that_does_not_exist_fails_its_channel(root, tmp_path):
    result = collect.collect(root, _command_channel([str(tmp_path / "nope.sh")]))
    assert not result.ok and "does not exist" in result.error


def test_a_command_result_skips_what_is_not_a_plain_allowed_file(root):
    channel = _command_channel(_script(
        "(out / 'sub').mkdir()\n(out / 'x.exe').write_bytes(b'x')\n(out / 'ok.md').write_bytes(b'ok')"))
    result = collect.collect(root, channel)
    assert [name for name, _ in result.taken] == ["ok.md"]
    assert dict(result.skipped) == {"sub": "not a regular file", "x.exe": "extension not allowed"}


# --- upload collect ---------------------------------------------------------

def _two_channels(root, drop):
    _write_intake(root, {
        "commands": {"scan": _script("(out / 'mail.md').write_bytes(b'mail\\n')")},
        "channels": {
            "onedrive": {"handler": "folder", "path": str(drop), "max_bytes": 1024,
                         "allowed_extensions": [".md"]},
            "mail": {"handler": "command", "command": "scan", "max_bytes": 1024,
                     "allowed_extensions": [".md"]},
            "server": {"handler": "mcp", "max_bytes": 1024, "allowed_extensions": [".md"],
                       "quota_submissions_per_day": 1, "quota_bytes_per_day": 1000},
        },
    })


def test_collect_without_a_pull_channel_has_nothing_to_do(root, capsys):
    upload_collect_command(names=None)
    assert "nothing to collect" in capsys.readouterr().out


def test_collect_pulls_every_pull_channel_and_reports_per_channel(root, drop, capsys):
    _two_channels(root, drop)
    _put(drop, "a.md", b"alpha\n")
    upload_collect_command(names=None)
    out = capsys.readouterr().out
    assert "onedrive (folder)" in out and "mail (command)" in out
    assert {m["channel"] for m in _manifests(root)} == {"onedrive", "mail"}


def test_collect_names_one_channel_and_leaves_the_others(root, drop):
    _two_channels(root, drop)
    _put(drop, "a.md", b"alpha\n")
    upload_collect_command(names=["mail"])
    assert {m["channel"] for m in _manifests(root)} == {"mail"}
    assert (drop / "a.md").exists()


def test_a_failing_channel_does_not_stop_the_others_but_the_exit_is_1(root, drop, tmp_path, capsys):
    _two_channels(root, drop)
    body = json.loads((root / config.LOCAL_SETTINGS_FILENAME).read_text())
    body["intake"]["channels"]["onedrive"]["path"] = str(tmp_path / "gone")
    (root / config.LOCAL_SETTINGS_FILENAME).write_text(json.dumps(body), encoding="utf-8")
    with pytest.raises(typer.Exit) as excinfo:
        upload_collect_command(names=None)
    assert excinfo.value.exit_code == 1
    assert "FAILED" in capsys.readouterr().out
    assert {m["channel"] for m in _manifests(root)} == {"mail"}


@pytest.mark.parametrize("names", [["nope"], ["server"], ["mail", "nope"], ["mail", "server"]])
def test_an_unknown_or_mcp_argument_exits_1_before_any_effect(root, drop, names):
    _two_channels(root, drop)
    _put(drop, "a.md", b"alpha\n")
    before = _tree(drop)
    with pytest.raises(typer.Exit) as excinfo:
        upload_collect_command(names=names)
    assert excinfo.value.exit_code == 1
    assert _tree(drop) == before
    assert _manifests(root) == []


def test_a_malformed_intake_section_exits_1(root):
    _write_intake(root, {"channels": {"a": {"handler": "folder"}}})
    with pytest.raises(typer.Exit) as excinfo:
        upload_collect_command(names=None)
    assert excinfo.value.exit_code == 1


def test_the_command_refuses_while_a_legacy_quarantine_holds_submissions(root, drop):
    _two_channels(root, drop)
    _put(drop, "a.md", b"alpha\n")
    (root / "mcp-upload" / "sub-1").mkdir(parents=True)
    (root / "mcp-upload" / "sub-1" / "a.md").write_bytes(b"waiting")
    with pytest.raises(typer.Exit) as excinfo:
        upload_collect_command(names=None)
    assert excinfo.value.exit_code == 1
    assert (drop / "a.md").exists()
    with pytest.raises(typer.Exit):
        upload_list_command(json_out=False)


# --- the quarantine's own .gitignore ----------------------------------------

def _git(root: Path, *args: str) -> str:
    return subprocess.run(["git", *args], cwd=root, check=True, capture_output=True, text=True).stdout


def test_collected_material_shows_in_no_git_status_even_without_a_root_entry(root, drop):
    _git(root, "init", "-q")
    _put(drop, "a.md", b"alpha\n")
    collect.collect(root, _folder_channel(drop))
    collect.collect(root, _command_channel(_script("(out / 'm.md').write_bytes(b'mail')")))
    assert (root / "quarantine" / ".gitignore").read_text().strip() == "*"
    assert _git(root, "status", "--porcelain", "--untracked-files=all") == ""


# --- backup and export data -------------------------------------------------

def _backup_target(root: Path, tmp_path: Path) -> Path:
    shared = tmp_path / "shared"
    (shared / "git").mkdir(parents=True)
    body = {"backup": {"target": str(shared), "machine_data": False, "machine_name": "WORK-LAPTOP"}}
    (root / config.LOCAL_SETTINGS_FILENAME).write_text(json.dumps({"schema": 1, **body}), encoding="utf-8")
    return shared


def _channel_at(root: Path, folder: Path) -> None:
    path = root / config.LOCAL_SETTINGS_FILENAME
    data = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {"schema": 1}
    data["intake"] = {"channels": {"inbox": {
        "handler": "folder", "path": str(folder), "max_bytes": 1024, "allowed_extensions": [".md"]}}}
    path.write_text(json.dumps(data), encoding="utf-8")


@pytest.mark.parametrize("where", ["git", "git/inner", "WORK-LAPTOP", "WORK-LAPTOP/inner"])
def test_an_intake_folder_equal_to_or_in_a_folder_the_backup_writes_is_refused(root, tmp_path, where):
    shared = _backup_target(root, tmp_path)
    _channel_at(root, shared / where)
    with pytest.raises(ValidationError, match="the backup's"):
        upload.read_channels(root)


def test_an_intake_folder_beside_or_over_the_backup_folders_is_valid(root, tmp_path):
    shared = _backup_target(root, tmp_path)
    for folder in (shared / "Inbox", shared, shared.parent):
        _channel_at(root, folder)
        assert upload.read_channels(root)["inbox"].path == folder


def test_a_broken_backup_section_does_not_break_the_intake_section(root, tmp_path):
    _backup_target(root, tmp_path)
    path = root / config.LOCAL_SETTINGS_FILENAME
    data = json.loads(path.read_text(encoding="utf-8"))
    data["backup"] = {"target": "relative/path"}
    path.write_text(json.dumps(data), encoding="utf-8")
    _channel_at(root, tmp_path / "shared" / "git")
    assert upload.read_channels(root)["inbox"].path == tmp_path / "shared" / "git"


def _backup_manifest(folder: Path) -> None:
    (folder / ".chemenu-backup.json").write_text("{}", encoding="utf-8")


def _bare_repository(folder: Path) -> None:
    (folder / "objects").mkdir()
    (folder / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8")


def _export_manifest(folder: Path) -> None:
    (folder / ".wikitool-export.json").write_text("{}", encoding="utf-8")


@pytest.mark.parametrize("mark, nested", [
    (_backup_manifest, False), (_backup_manifest, True),
    (_bare_repository, False), (_bare_repository, True),
    (_export_manifest, False), (_export_manifest, True),
], ids=["machine", "in-machine", "bare", "in-bare", "export", "in-export"])
def test_a_pull_from_backup_or_export_data_fails_its_channel_and_leaves_the_folder_as_it_was(
        root, drop, mark, nested):
    base = drop / "inner" if nested else drop
    base.mkdir(exist_ok=True)
    mark(drop)
    _put(base, "ENVIRONMENT.md", b"machine data\n")
    before = _tree(drop)
    result = collect.collect(root, _folder_channel(base))
    assert result.error and "lies in" in result.error
    assert not (result.taken or result.failed or result.skipped)
    assert _tree(drop) == before
    assert _manifests(root) == []


def test_the_other_channels_still_run_when_one_lies_in_backup_data(root, drop, tmp_path, capsys):
    _two_channels(root, drop)
    _backup_manifest(drop)
    _put(drop, "a.md", b"alpha\n")
    before = _tree(drop)
    with pytest.raises(typer.Exit) as excinfo:
        upload_collect_command(names=None)
    assert excinfo.value.exit_code == 1
    assert _tree(drop) == before
    assert {m["channel"] for m in _manifests(root)} == {"mail"}