chemenu/tools/chemenu/legacy_settings.py

"""The four per-feature settings files of earlier versions, and the move out of them.

Before the tiered settings (`chemenu.settings`) every feature had a file of its
own at the root of the checkout. None of them is read any more; this is the one
module that still names them, to detect a leftover and to carry its content
into `wikitool.json`, `.wikitool-local.json` and the secrets file -
`wikitool config migrate`.

| Old file | Becomes |
|---|---|
| `.wikitool-telemetry.json` | section `telemetry` |
| `.wikitool-remotes.json` | section `remotes` |
| `.wikitool-upload.json` | section `intake`, channel `mcp` with `handler: "mcp"` (`quota.*` is flat: `quota_*`) |
| `.wikitool-tasks.json` | sections `tasks` (`thresholds.*` is flat), `superproductivity`, `caldav` |

What the migration promises, and how it keeps it:

- **Every setting keeps its effective value, defaults included.** The old
  content is read with the same coercions and checks the new sections use
  (`config_shapes.check`), so a value is accepted exactly when it was, and what
  is written is the coerced value. A field the old file did not set is not
  written: its default still applies.
- **No secret value reaches the tree.** A secret field's value goes into the
  secrets file (`config.secrets_file`), only a reference into `wikitool.json`;
  nothing here prints a value.
- **An abort leaves the old files as they were.** Everything is computed and
  checked in memory first. Then the targets are written - secrets file,
  `wikitool.json`, `.wikitool-local.json` - and read back and checked again.
  Only then are the old files removed; before that, any failure puts every
  target back to the bytes it had.
"""
from __future__ import annotations

import copy
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Optional

from chemenu import config, config_shapes, settings
from chemenu.config_shapes import INSTANCE, SECRET
from chemenu.errors import ChemenuError, ValidationError

TELEMETRY_FILE = ".wikitool-telemetry.json"
REMOTES_FILE = ".wikitool-remotes.json"
UPLOAD_FILE = ".wikitool-upload.json"
TASKS_FILE = ".wikitool-tasks.json"

# Feature (= the section that replaces the file) -> the file it came from.
FILES = {
    "telemetry": TELEMETRY_FILE,
    "remotes": REMOTES_FILE,
    "intake": UPLOAD_FILE,
    "tasks": TASKS_FILE,
}

MIGRATE_COMMAND = "tools/wikitool config migrate"


class LegacySettingsLeftover(ChemenuError):
    """A settings file of an earlier version is still there and nothing reads
    it. Not a fault in the caller's input and not something to retry: a human
    runs `config migrate`, so the CLI renders it as exit 42 (`needs_clearance`)."""

    def __init__(self, names: list[str]):
        self.names = names
        listed = ", ".join(names)
        verb = "is a settings file" if len(names) == 1 else "are settings files"
        super().__init__(
            f"{listed} {verb} of an earlier version and no longer read - the settings it held "
            f"are not in effect. Move them into the tiered settings with: {MIGRATE_COMMAND}"
        )


def present(root: "Path | str") -> list[str]:
    """The old files that exist under `root`, in a fixed order."""
    return [name for name in FILES.values() if (Path(root) / name).exists()]


def refuse_leftover(root: "Path | str", *features: str) -> None:
    """Raise `LegacySettingsLeftover` when the old file of any of `features`
    exists. The tracker file is not looked at while a tracker profile
    (`WIKITOOL_TASKS_CONFIG`) replaces its sections - it is not what is read."""
    names = []
    for feature in features:
        if feature == "tasks" and config.tasks_config_override() is not None:
            continue
        name = FILES[feature]
        if (Path(root) / name).exists():
            names.append(name)
    if names:
        raise LegacySettingsLeftover(names)


# ---------------------------------------------------------------------------
# Reading the old files
# ---------------------------------------------------------------------------


@dataclass
class Plan:
    """What the migration would do: the values per section (coerced, JSON-safe),
    and what to say about them."""

    entries: dict[str, dict[str, Any]] = field(default_factory=dict)
    notes: list[str] = field(default_factory=list)
    files: list[Path] = field(default_factory=list)


def _json_safe(value: Any) -> Any:
    return list(value) if isinstance(value, tuple) else value


def _read_old(path: Path, *, lenient: bool, plan: Plan) -> Any:
    """The old file's parsed JSON. A telemetry file that cannot be read counts
    as absent, as it always did; any other raises `ValidationError`."""
    import json

    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except (OSError, ValueError) as exc:
        if lenient:
            plan.notes.append(f"{path.name} could not be read ({type(exc).__name__}) and counts as absent, as before")
            return None
        raise ValidationError(f"{path.name} cannot be read as JSON ({exc}) - fix or remove it, then run again") from None


def _take(plan: Plan, shape: config_shapes.Shape, data: Any, *, file: str, lenient: bool = False) -> None:
    """The coerced values of `data` that `shape` accepts, into `plan`. Strict:
    a problem raises `ValidationError` naming the old file. Lenient: a field
    with a problem is dropped and said so."""
    checked = config_shapes.check(shape, data)
    if checked.problems:
        found = "; ".join(problem.message for problem in checked.problems)
        if not lenient:
            raise ValidationError(f"{file}: {found}")
        plan.notes.append(f"{file}: not migrated, as it was not read - {found}")
    values = {name: _json_safe(checked.values[name]) for name in checked.given}
    ignored = [key for key in checked.unknown if key != "schema"]
    if ignored:
        plan.notes.append(f"{file}: {', '.join(ignored)} was not read before and is not migrated")
    if values:
        plan.entries.setdefault(shape.key, {}).update(values)


def _flat(data: Any, nested: str, prefix: str, file: str) -> Any:
    """`data` with the object `nested` spread into `<prefix>_<key>` keys."""
    if not isinstance(data, dict) or nested not in data:
        return data
    inner = data[nested]
    if not isinstance(inner, dict):
        raise ValidationError(f"{file}: {nested} must be an object")
    flat = {key: value for key, value in data.items() if key != nested}
    for key, value in inner.items():
        flat[f"{prefix}{key}"] = value
    return flat


def _provider_section(name: str, body: Any, file: str) -> dict[str, Any]:
    """The migrated values of one provider section, after the checks the
    adapter makes when it reads it - the access-path rules of Super
    Productivity included. Raises `ValidationError`."""
    shape = {config_shapes.SUPERPRODUCTIVITY.key: config_shapes.SUPERPRODUCTIVITY,
             config_shapes.CALDAV.key: config_shapes.CALDAV}[name]
    if not isinstance(body, dict):
        raise ValidationError(f"{file}: {name} must be an object")
    try:
        section = settings.Section.from_data(shape, body)
        if name == config_shapes.SUPERPRODUCTIVITY.key:
            from chemenu.tasks import superproductivity

            superproductivity.SuperProductivityConfig.from_section(section)
    except ValidationError as exc:
        raise ValidationError(f"{file}: {name}: {exc}") from None
    return {key: _json_safe(section.values[key]) for key in section.given}


def read_plan(root: "Path | str") -> Plan:
    """What `config migrate` would move, from every old file under `root`.
    Raises `ValidationError` for the first old file that is broken; nothing is
    written by then."""
    root = Path(root)
    plan = Plan()

    path = root / TELEMETRY_FILE
    if path.exists():
        plan.files.append(path)
        data = _read_old(path, lenient=True, plan=plan)
        if data is not None and not isinstance(data, dict):
            plan.notes.append(f"{TELEMETRY_FILE} is not a JSON object and counts as absent, as before")
        elif data is not None:
            _take(plan, config_shapes.TELEMETRY, data, file=TELEMETRY_FILE, lenient=True)

    path = root / REMOTES_FILE
    if path.exists():
        plan.files.append(path)
        _take(plan, config_shapes.REMOTES, _read_old(path, lenient=False, plan=plan), file=REMOTES_FILE)

    path = root / UPLOAD_FILE
    if path.exists():
        plan.files.append(path)
        data = _flat(_read_old(path, lenient=False, plan=plan), "quota", "quota_", UPLOAD_FILE)
        if isinstance(data, dict):
            known = set(config_shapes.mcp_channel_keys())
            ignored = [key for key in data if key not in known and key != "schema"]
            if ignored:
                plan.notes.append(f"{UPLOAD_FILE}: {', '.join(ignored)} was not read before and is not migrated")
            data = {key: value for key, value in data.items() if key in known}
            data = {"handler": config_shapes.INTAKE_HANDLER_MCP, **data}
        _take(plan, config_shapes.INTAKE, {"channels": {"mcp": data}}, file=UPLOAD_FILE)

    path = root / TASKS_FILE
    if path.exists():
        plan.files.append(path)
        _read_tasks(path, plan)
    return plan


def _read_tasks(path: Path, plan: Plan) -> None:
    data = _read_old(path, lenient=False, plan=plan)
    data = _flat(data, "thresholds", "", TASKS_FILE)
    # `thresholds` was an object whose keys are now the section's own fields;
    # `provider` was the only other field. Everything else is a provider section.
    _take(plan, config_shapes.TASKS, {k: v for k, v in data.items() if k not in config_shapes.PROVIDERS}
          if isinstance(data, dict) else data, file=TASKS_FILE)
    provider = plan.entries["tasks"]["provider"]
    for name in config_shapes.PROVIDERS:
        body = data.get(name)
        if name == provider:
            if body is None:
                raise ValidationError(f"{TASKS_FILE}: provider is {provider!r}, so the file needs a {provider} section")
            plan.entries[name] = _provider_section(name, body, TASKS_FILE)
        elif body is not None:
            try:
                plan.entries[name] = _provider_section(name, body, TASKS_FILE)
            except ValidationError as exc:
                plan.notes.append(f"{exc} - the section of a provider that is not chosen is not migrated")


# ---------------------------------------------------------------------------
# Splitting by tier, merging into what is there
# ---------------------------------------------------------------------------


@dataclass
class Targets:
    instance: dict
    local: dict
    secrets: dict
    moved: list[str]


def _secret_name(section: str, name: str) -> str:
    return f"{section}_{name}"


def _merge(target: dict, section: Optional[str], name: str, value: Any, label: str) -> None:
    body = target
    where = name
    if section is not None:
        body = target.setdefault(section, {})
        where = f"{section}.{name}"
        if not isinstance(body, dict):
            raise ValidationError(f"{label}: {section} is not an object")
    if name in body and body[name] != value:
        raise ValidationError(
            f"{label}: {where} already holds a different value than the old file. Decide which "
            "one stands, remove the other, and run again"
        )
    body[name] = value


def split(plan: Plan, instance: dict, local: dict, secrets: dict) -> Targets:
    """`plan` distributed over the files by tier, on top of what they hold.
    Raises `ValidationError` on a key that already holds another value."""
    instance, local, secrets = copy.deepcopy(instance), copy.deepcopy(local), dict(secrets)
    known = settings.sections()
    moved: list[str] = []
    for section, values in plan.entries.items():
        shape = known[section]
        for name, value in values.items():
            spec = shape.field(name)
            key = f"{section}.{name}"
            if spec.tier == SECRET:
                ref = _secret_name(section, name)
                if secrets.get(ref, value) != value:
                    raise ValidationError(
                        f"the secrets file already holds a different value under {ref!r} - decide "
                        "which one stands, remove the other, and run again"
                    )
                secrets[ref] = value
                _merge(instance, section, name, {settings.REF_KEY: ref}, config.INSTANCE_SETTINGS_FILENAME)
                moved.append(f"{key} -> secrets file, reference {ref!r} in {config.INSTANCE_SETTINGS_FILENAME}")
            elif spec.tier == INSTANCE:
                _merge(instance, section, name, value, config.INSTANCE_SETTINGS_FILENAME)
                moved.append(f"{key} -> {config.INSTANCE_SETTINGS_FILENAME}")
            else:
                _merge(local, section, name, value, config.LOCAL_SETTINGS_FILENAME)
                moved.append(f"{key} -> {config.LOCAL_SETTINGS_FILENAME}")
    return Targets(instance, local, secrets, moved)


# ---------------------------------------------------------------------------
# Writing, reading back, rolling back
# ---------------------------------------------------------------------------


def _write(path: Path, data: dict) -> None:
    settings.write_file(path, data)


def _missing_ancestors(path: Path) -> list[Path]:
    found = []
    parent = path.parent
    while not parent.exists():
        found.append(parent)
        parent = parent.parent
    return found


def _restore(originals: dict[Path, Optional[bytes]], created_dirs: list[Path]) -> None:
    for path, content in originals.items():
        try:
            if content is None:
                path.unlink(missing_ok=True)
            else:
                path.write_bytes(content)
        except OSError:
            pass
    for directory in created_dirs:
        try:
            directory.rmdir()
        except OSError:
            pass


def _normal(value: Any) -> Any:
    return list(value) if isinstance(value, tuple) else value


def _verify(root: Path, plan: Plan, secrets_path: Optional[Path], expected_secrets: dict) -> None:
    """Read the written files back and check that every planned value is in
    effect. Raises `ValidationError`."""
    loaded = settings.check_files(settings.read_file(settings.instance_path(root)),
                                  settings.read_file(settings.local_path(root)))
    known = settings.sections()
    for section, values in plan.entries.items():
        got = loaded.section(section)
        for name, value in values.items():
            spec = known[section].field(name)
            if spec.tier == SECRET:
                ref = got.refs.get(name)
                if ref is None or ref.target != _secret_name(section, name):
                    raise ValidationError(f"read-back: {section}.{name} is not a reference to the secrets file")
            elif _normal(got.values.get(name)) != value:
                raise ValidationError(f"read-back: {section}.{name} does not hold the migrated value")
    if expected_secrets:
        assert secrets_path is not None
        stored = settings.read_secrets_file(secrets_path) or {}
        for ref, value in expected_secrets.items():
            if stored.get(ref) != value:
                raise ValidationError(f"read-back: the secrets file does not hold {ref!r}")


@dataclass
class Result:
    moved: list[str]
    notes: list[str]
    removed: list[str]


def migrate(root: "Path | str") -> Result:
    """Carry every old file under `root` into the tiered settings. Raises
    `ValidationError` - with the old files and the targets as they were - for
    anything it cannot do or check; returns an empty `Result` when there is
    nothing to move."""
    root = Path(root)
    plan = read_plan(root)
    if not plan.files:
        return Result([], [], [])

    instance_path, local_path = settings.instance_path(root), settings.local_path(root)
    instance = settings.read_file(instance_path)
    local = settings.read_file(local_path)

    # Everything that can be known without writing is checked first.
    instance_id = (instance or {}).get(settings.INSTANCE_ID_KEY)
    needs_secrets = any(_tier(section, name) == SECRET
                        for section, values in plan.entries.items() for name in values)
    secrets_path: Optional[Path] = None
    existing_secrets: dict = {}
    if needs_secrets:
        if instance_id is None:
            instance_id = settings.new_instance_id()
        secrets_path = config.secrets_file(instance_id)
        existing_secrets = settings.read_secrets_file(secrets_path) or {}

    targets = split(plan, instance or {}, local or {}, existing_secrets)
    changed_secrets = {ref: value for ref, value in targets.secrets.items()
                       if existing_secrets.get(ref) != value}
    write_instance = targets.instance != (instance or {})
    write_local = targets.local != (local or {})
    if write_instance:
        targets.instance.setdefault("schema", settings.SCHEMA)
        targets.instance.setdefault(settings.INSTANCE_ID_KEY, instance_id or settings.new_instance_id())
    if write_local:
        targets.local.setdefault("schema", settings.SCHEMA)
    try:
        settings.check_files(targets.instance if (write_instance or instance is not None) else None,
                             targets.local if (write_local or local is not None) else None)
    except ValidationError as exc:
        raise ValidationError(f"the settings would not pass their check after the move:\n{exc}") from None

    writes: list[tuple[Path, dict]] = []
    if changed_secrets:
        assert secrets_path is not None
        writes.append((secrets_path, targets.secrets))
    if write_instance:
        writes.append((instance_path, targets.instance))
    if write_local:
        writes.append((local_path, targets.local))

    originals = {path: (path.read_bytes() if path.exists() else None) for path, _ in writes}
    created_dirs = _missing_ancestors(secrets_path) if changed_secrets and secrets_path else []
    try:
        for path, data in writes:
            _write(path, data)
        _verify(root, plan, secrets_path, {ref: targets.secrets[ref] for ref in changed_secrets})
    except BaseException as exc:
        _restore(originals, created_dirs)
        if isinstance(exc, ValidationError):
            raise ValidationError(f"{exc}. Nothing was changed; the old files are as they were") from None
        if isinstance(exc, Exception):
            raise ValidationError(
                f"the move failed ({type(exc).__name__}). Nothing was changed; the old files are as they were"
            ) from None
        raise

    removed = []
    for path in plan.files:
        path.unlink()
        removed.append(path.name)
    return Result(targets.moved, plan.notes, removed)


def _tier(section: str, name: str) -> str:
    return settings.sections()[section].field(name).tier