torbenandClaude Sonnet 5.5
02bf3b0403
Files changed: - CHANGES.md - VERSION - docs/how-to/back-up-to-a-synced-folder.md - docs/how-to/collect-from-a-synced-folder.md - docs/how-to/export-to-a-synced-folder.md - docs/reference/commands/exports.md - docs/reference/commands/raw-material-and-uploads.md - docs/reference/configuration.md - tools/CONTRACT.md - tools/chemenu/backup.py - tools/chemenu/collect.py - tools/chemenu/commands/export_cmd.py - tools/chemenu/commands/upload_cmd.py - tools/chemenu/config_shapes.py - tools/chemenu/page_export.py - tools/chemenu/tests/test_collect.py - tools/chemenu/tests/test_export_run.py - tools/chemenu/upload.py Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
605 lines
24 KiB
Python
605 lines
24 KiB
Python
"""Backing an instance up into a folder a sync client uploads (OneDrive and the like).
|
|
|
|
The working tree stays out of that folder: a sync client locks files under
|
|
`.git` for a moment (which breaks `git gc` and `repack`), offloads them as
|
|
placeholders, and makes conflict copies inside `.git` as soon as a second device
|
|
syncs the same folder. What goes into the synced folder is a backup target with
|
|
this layout (`config_shapes.BACKUP` is the section that names it):
|
|
|
|
<target>/git/ a bare repository, the checkout's `origin`; `publish` pushes to it
|
|
<target>/<machine>/ a mirror of this machine's files that are not in git,
|
|
plus the manifest `.chemenu-backup.json`
|
|
|
|
Git writes into the bare repository only on a push, with one writer, so the
|
|
sync client sees mostly new, immutable object files.
|
|
|
|
What a machine-data run guarantees - each one has its test:
|
|
|
|
- It writes and deletes only under `<target>/<machine>/`. Nothing outside it
|
|
changes - not `<target>/git`, not another machine's folder.
|
|
- It deletes only what the previous manifest lists and is gone locally. A file
|
|
in the folder the manifest does not list stays as it is.
|
|
- It refuses a non-empty `<target>/<machine>/` without a manifest, before it
|
|
writes anything: it never mirrors into a folder it did not write.
|
|
- It reads nothing outside the instance. An `extra` path that resolves outside
|
|
is an error of the section; a symbolic link (or junction) inside a mirrored
|
|
directory is skipped and reported, never followed; `.git` is never mirrored.
|
|
- Every file is written atomically - a temporary file in the same directory,
|
|
then `os.replace` - and the manifest last, so an interrupted run leaves whole
|
|
files and the previous run's manifest.
|
|
|
|
The secrets file lives outside the instance (`config.secrets_file`), so the last
|
|
two rules keep it out of the synced folder as well.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import socket
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
from chemenu import config, config_shapes, settings, toolpaths
|
|
from chemenu.errors import ValidationError
|
|
|
|
MANIFEST_NAME = ".chemenu-backup.json"
|
|
MANIFEST_SCHEMA = 1
|
|
INCOMING_DIR = "incoming"
|
|
INCOMING_KEEP = ".gitkeep"
|
|
GIT_NAME = ".git"
|
|
# The files at the root of the instance a machine-data run mirrors when they exist.
|
|
ROOT_FILES = (config.LOCAL_SETTINGS_FILENAME, config.ENVIRONMENT_FILE)
|
|
|
|
# Where a sync client keeps its folder: the variables the OneDrive client sets on
|
|
# Windows, and the folder macOS's File Provider puts every sync client under.
|
|
SYNC_ENV_VARS = ("OneDrive", "OneDriveCommercial", "OneDriveConsumer")
|
|
MAC_CLOUD_STORAGE = ("Library", "CloudStorage")
|
|
# Windows compares paths without regard to case; a test sets this to compare
|
|
# that way on another platform.
|
|
IGNORE_CASE = os.name == "nt"
|
|
|
|
# OneDrive holds a file it is uploading open for a moment; `os.replace` onto it
|
|
# then fails with a sharing violation that is gone a second later.
|
|
_REPLACE_ATTEMPTS = 5
|
|
_REPLACE_PAUSE_SECONDS = 0.4
|
|
|
|
|
|
class BackupError(ValidationError):
|
|
"""A backup step refused or failed; exit 1 with the message."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BackupConfig:
|
|
target: Path
|
|
machine_data: bool
|
|
machine_name: str
|
|
extra: tuple[str, ...]
|
|
|
|
@property
|
|
def git_dir(self) -> Path:
|
|
return self.target / config_shapes.BACKUP_GIT_DIR
|
|
|
|
@property
|
|
def machine_dir(self) -> Path:
|
|
return self.target / self.machine_name
|
|
|
|
@property
|
|
def manifest(self) -> Path:
|
|
return self.machine_dir / MANIFEST_NAME
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Paths
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def is_link(path: Path) -> bool:
|
|
is_junction = getattr(path, "is_junction", None) # Python 3.12+
|
|
return path.is_symlink() or bool(is_junction and is_junction())
|
|
|
|
|
|
def real_path(path: Path) -> Path:
|
|
return Path(os.path.realpath(path))
|
|
|
|
|
|
def is_under(path: Path, root: Path, ignore_case: Optional[bool] = None) -> bool:
|
|
"""Whether `path` is `root` or lies below it, both taken as given."""
|
|
ignore_case = IGNORE_CASE if ignore_case is None else ignore_case
|
|
inner, outer = str(path), str(root).rstrip("/\\")
|
|
if ignore_case:
|
|
inner, outer = inner.casefold(), outer.casefold()
|
|
return inner == outer or inner.startswith(outer + os.sep) or inner.startswith(outer + "/")
|
|
|
|
|
|
def _names_git(parts) -> bool:
|
|
return any(part.casefold() == GIT_NAME for part in parts)
|
|
|
|
|
|
def safe_relative(rel: str) -> bool:
|
|
"""Whether `rel`, a `/`-separated path, stays below the folder it is joined
|
|
to on every platform: no empty, `.` or `..` part, no backslash, no drive."""
|
|
parts = rel.split("/")
|
|
return bool(rel) and all(part not in ("", ".", "..") and "\\" not in part and ":" not in part
|
|
for part in parts)
|
|
|
|
|
|
def _within(machine_dir: Path, rel: str) -> Path:
|
|
"""`rel` below `machine_dir`, refusing a path that would leave it - by its
|
|
spelling, or through a link somewhere in the folder."""
|
|
if not safe_relative(rel):
|
|
raise BackupError(f"{rel!r} is not a path below {machine_dir} - nothing further was written")
|
|
dest = machine_dir.joinpath(*rel.split("/"))
|
|
current = machine_dir
|
|
for part in rel.split("/")[:-1]:
|
|
current = current / part
|
|
if is_link(current):
|
|
raise BackupError(f"{current} is a link - a backup writes nothing through one")
|
|
return dest
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Reading the section
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _extra_problem(root: Path, real_root: Path, entry: str) -> Optional[str]:
|
|
if _names_git(Path(entry).parts):
|
|
return f"backup.extra holds {entry!r}, which names {GIT_NAME}/ - it is never backed up"
|
|
try:
|
|
resolved = (root / entry).resolve()
|
|
except (OSError, RuntimeError) as exc:
|
|
return f"backup.extra holds {entry!r}, which cannot be resolved ({exc})"
|
|
if not is_under(resolved, real_root):
|
|
return (f"backup.extra holds {entry!r}, which resolves to {resolved}, outside the "
|
|
"instance - a backup reads nothing outside it")
|
|
if _names_git(resolved.relative_to(real_root).parts):
|
|
return f"backup.extra holds {entry!r}, which resolves into {GIT_NAME}/ - it is never backed up"
|
|
return None
|
|
|
|
|
|
def read_config(root: "Path | str | None" = None) -> Optional[BackupConfig]:
|
|
"""The `backup` section, or None when the settings carry none. Raises
|
|
`ValidationError` for a section that is broken - including what the record
|
|
cannot judge without the instance: a target inside it, an `extra` path that
|
|
leaves it, a host name that cleans to nothing."""
|
|
root = Path(config.ROOT if root is None else root)
|
|
key = config_shapes.BACKUP.key
|
|
section = settings.load(root, only=(key,)).section(key)
|
|
if not section.configured:
|
|
return None
|
|
problems = []
|
|
real_root = real_path(root)
|
|
|
|
target = Path(section["target"])
|
|
if not os.path.isabs(section["target"]):
|
|
problems.append(f"backup.target {section['target']!r} is not an absolute path on this machine")
|
|
else:
|
|
real_target = real_path(target)
|
|
if is_under(real_target, real_root):
|
|
problems.append(f"backup.target {target} lies inside the instance - it belongs outside it")
|
|
|
|
if "machine_name" in section.given:
|
|
name, origin = section["machine_name"], ""
|
|
else:
|
|
name, origin = socket.gethostname(), " (the host name, since machine_name is not set)"
|
|
problem = config_shapes.machine_name_problem(name)
|
|
if problem:
|
|
problems.append(f"backup.machine_name {name!r}{origin} {problem} - set machine_name")
|
|
machine_name = config_shapes.machine_name(name)
|
|
|
|
if not problems:
|
|
for inner in (target / config_shapes.BACKUP_GIT_DIR, target / machine_name):
|
|
if is_under(real_root, real_path(inner)):
|
|
problems.append(f"the instance lies inside {inner}, which the backup writes to")
|
|
|
|
for entry in section["extra"]:
|
|
found = _extra_problem(root, real_root, entry)
|
|
if found:
|
|
problems.append(found)
|
|
|
|
if problems:
|
|
raise ValidationError("; ".join(problems))
|
|
return BackupConfig(target=target, machine_data=section["machine_data"],
|
|
machine_name=machine_name, extra=tuple(section["extra"]))
|
|
|
|
|
|
def written_folders(root: "Path | str | None" = None) -> list[tuple[str, Path]]:
|
|
"""The folders this instance's backup writes into - `<target>/git` and
|
|
`<target>/<own machine>` - each with a label for a message. Whatever else
|
|
lies in `<target>` is the sync folder's, not the backup's, so an export or
|
|
an intake folder may sit beside these two. Read as leniently as the other
|
|
features read each other's folders: a broken section yields none, it is
|
|
reported where it is read (`doctor`'s `backup` check)."""
|
|
try:
|
|
cfg = read_config(root)
|
|
except ValidationError:
|
|
return []
|
|
if cfg is None:
|
|
return []
|
|
return [(f"the backup's git folder ({cfg.git_dir})", cfg.git_dir),
|
|
(f"the backup's machine folder ({cfg.machine_dir})", cfg.machine_dir)]
|
|
|
|
|
|
def _is_bare_repo_dir(folder: Path) -> bool:
|
|
return (folder / "HEAD").is_file() and (folder / "objects").is_dir()
|
|
|
|
|
|
def data_marker(folder: Path, extra_manifests: tuple[str, ...] = ()) -> Optional[str]:
|
|
"""What marks `folder`, or one of its ancestors, as data that belongs to the
|
|
backup - a machine folder (the backup manifest), a bare repository - or to
|
|
a manifest named in `extra_manifests`; None when nothing does. The
|
|
configuration cannot see another machine's folder, or one a machine starts
|
|
to fill later, so a run asks the folder itself; the ancestors cover a
|
|
folder inside one. Looked at through links, and without a `git` call."""
|
|
resolved = real_path(folder)
|
|
for candidate in (resolved, *resolved.parents):
|
|
try:
|
|
for name in (MANIFEST_NAME, *extra_manifests):
|
|
if (candidate / name).is_file():
|
|
return f"{candidate} holds {name}"
|
|
if _is_bare_repo_dir(candidate):
|
|
return f"{candidate} is a git repository"
|
|
except OSError:
|
|
continue
|
|
return None
|
|
|
|
|
|
def require_config(root: "Path | str | None" = None) -> BackupConfig:
|
|
found = read_config(root)
|
|
if found is None:
|
|
raise BackupError(
|
|
f"no backup section in {config.LOCAL_SETTINGS_FILENAME} - the user adds it by hand "
|
|
"(docs/how-to/back-up-to-a-synced-folder.md)"
|
|
)
|
|
return found
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# git
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _git(args: list[str], cwd: Path) -> subprocess.CompletedProcess:
|
|
return subprocess.run([toolpaths.git(), *args], cwd=cwd, capture_output=True, text=True,
|
|
encoding="utf-8", errors="replace")
|
|
|
|
|
|
def is_bare_repo(path: Path) -> bool:
|
|
if not path.is_dir():
|
|
return False
|
|
result = _git(["--git-dir", str(path), "rev-parse", "--is-bare-repository"], cwd=path)
|
|
return result.returncode == 0 and result.stdout.strip() == "true"
|
|
|
|
|
|
def push_url(root: Path, remote: str = "origin") -> Optional[str]:
|
|
result = _git(["remote", "get-url", "--push", remote], cwd=root)
|
|
if result.returncode != 0:
|
|
return None
|
|
return result.stdout.strip() or None
|
|
|
|
|
|
def same_location(url: str, path: Path, root: Path) -> bool:
|
|
"""Whether the remote URL `url` names the directory `path`. A URL with a
|
|
scheme other than `file://`, or an scp-like `host:path`, never does."""
|
|
local = url[len("file://"):] if url.startswith("file://") else url
|
|
if "://" in local or (":" in local and not os.path.isabs(local)):
|
|
return False
|
|
a, b = os.path.normcase(real_path(root / local)), os.path.normcase(real_path(path))
|
|
return a.casefold() == b.casefold() if IGNORE_CASE else a == b
|
|
|
|
|
|
def _directory_state(path: Path) -> str:
|
|
if not path.exists() and not is_link(path):
|
|
return "absent"
|
|
if path.is_dir() and not is_link(path) and not any(path.iterdir()):
|
|
return "empty"
|
|
return "bare" if is_bare_repo(path) else "other"
|
|
|
|
|
|
@dataclass
|
|
class InitResult:
|
|
git_dir: Path
|
|
created_repo: bool
|
|
set_origin: bool
|
|
allowlist_entry: Optional[str]
|
|
|
|
|
|
def allowlist_entry(root: Path, url: str) -> Optional[str]:
|
|
"""The `remotes` section as it would read with `url` added, when it exists
|
|
and lacks `url` - printed for the user to copy, never written: the field is
|
|
a gate."""
|
|
key = config_shapes.REMOTES.key
|
|
section = settings.load(root, only=(key,)).section(key)
|
|
if not section.configured:
|
|
return None
|
|
allowed = list(section["allowed_push_urls"])
|
|
if url in allowed:
|
|
return None
|
|
return json.dumps({key: {"allowed_push_urls": [*allowed, url]}}, indent=2, ensure_ascii=False)
|
|
|
|
|
|
def init(cfg: BackupConfig, root: "Path | str | None" = None) -> InitResult:
|
|
"""Create `<target>/git` as a bare repository and make it `origin` when
|
|
there is none. Checks everything first and changes nothing on a refusal."""
|
|
root = Path(config.ROOT if root is None else root)
|
|
state = _directory_state(cfg.git_dir)
|
|
if state == "other":
|
|
raise BackupError(
|
|
f"{cfg.git_dir} exists and is not a bare git repository - nothing was changed. Move it "
|
|
"away, or point backup.target at another folder"
|
|
)
|
|
current = push_url(root)
|
|
if current is not None and not same_location(current, cfg.git_dir, root):
|
|
raise BackupError(
|
|
f"origin already pushes to {current}, not to {cfg.git_dir} - nothing was changed. "
|
|
f"Whether origin moves is the user's decision: git remote set-url origin \"{cfg.git_dir}\""
|
|
)
|
|
|
|
created = False
|
|
if state != "bare":
|
|
cfg.git_dir.mkdir(parents=True, exist_ok=True)
|
|
result = _git(["init", "--bare", "--quiet", "-b", "main", str(cfg.git_dir)], cwd=root)
|
|
if result.returncode != 0:
|
|
raise BackupError(f"git init --bare {cfg.git_dir} failed:\n{result.stderr}")
|
|
created = True
|
|
|
|
set_origin = False
|
|
if current is None:
|
|
result = _git(["remote", "add", "origin", str(cfg.git_dir)], cwd=root)
|
|
if result.returncode != 0:
|
|
raise BackupError(f"git remote add origin failed:\n{result.stderr}")
|
|
set_origin, current = True, str(cfg.git_dir)
|
|
return InitResult(cfg.git_dir, created, set_origin, allowlist_entry(root, current))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Machine data
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class Collected:
|
|
files: dict = field(default_factory=dict) # relative posix path -> source Path
|
|
skipped: list = field(default_factory=list) # "<path> (<why>)"
|
|
|
|
|
|
def _walk(path: Path, rel: str, out: Collected) -> None:
|
|
if is_link(path):
|
|
out.skipped.append(f"{rel} (a link - not followed)")
|
|
return
|
|
if path.name.casefold() == GIT_NAME:
|
|
return
|
|
if path.is_file():
|
|
if rel == MANIFEST_NAME:
|
|
out.skipped.append(f"{rel} (the manifest's own name)")
|
|
elif not safe_relative(rel):
|
|
out.skipped.append(f"{rel} (a name that cannot be mirrored on every platform)")
|
|
else:
|
|
out.files.setdefault(rel, path)
|
|
return
|
|
if path.is_dir():
|
|
for child in sorted(path.iterdir()):
|
|
_walk(child, f"{rel}/{child.name}" if rel else child.name, out)
|
|
|
|
|
|
def collect(cfg: BackupConfig, root: "Path | str | None" = None) -> Collected:
|
|
"""What a machine-data run mirrors: the root files that exist, everything in
|
|
`incoming/` but its `.gitkeep`, and `extra`."""
|
|
root = Path(config.ROOT if root is None else root)
|
|
real_root = real_path(root)
|
|
out = Collected()
|
|
for name in ROOT_FILES:
|
|
if (root / name).exists() or is_link(root / name):
|
|
_walk(root / name, name, out)
|
|
incoming = root / INCOMING_DIR
|
|
if is_link(incoming):
|
|
out.skipped.append(f"{INCOMING_DIR} (a link - not followed)")
|
|
elif incoming.is_dir():
|
|
for child in sorted(incoming.iterdir()):
|
|
if child.name != INCOMING_KEEP:
|
|
_walk(child, f"{INCOMING_DIR}/{child.name}", out)
|
|
for entry in cfg.extra:
|
|
resolved = (root / entry).resolve()
|
|
if not is_under(resolved, real_root): # read_config refuses this; never read it anyway
|
|
raise BackupError(f"backup.extra {entry!r} resolves outside the instance")
|
|
if not resolved.exists():
|
|
out.skipped.append(f"{entry} (not present)")
|
|
continue
|
|
rel = resolved.relative_to(real_root).as_posix()
|
|
_walk(resolved, "" if rel == "." else rel, out)
|
|
return out
|
|
|
|
|
|
def file_digest(path: Path) -> tuple[int, str]:
|
|
sha, size = hashlib.sha256(), 0
|
|
with open(path, "rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1 << 20), b""):
|
|
sha.update(chunk)
|
|
size += len(chunk)
|
|
return size, sha.hexdigest()
|
|
|
|
|
|
def _replace(tmp: str, dest: Path) -> None:
|
|
for attempt in range(_REPLACE_ATTEMPTS):
|
|
try:
|
|
os.replace(tmp, dest)
|
|
return
|
|
except PermissionError:
|
|
if attempt == _REPLACE_ATTEMPTS - 1:
|
|
raise
|
|
time.sleep(_REPLACE_PAUSE_SECONDS)
|
|
|
|
|
|
def write_atomic(dest: Path, chunks) -> tuple[int, str]:
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
fd, tmp = tempfile.mkstemp(prefix=".chemenu-", suffix=".tmp", dir=str(dest.parent))
|
|
sha, size = hashlib.sha256(), 0
|
|
try:
|
|
with os.fdopen(fd, "wb") as out:
|
|
for chunk in chunks:
|
|
out.write(chunk)
|
|
sha.update(chunk)
|
|
size += len(chunk)
|
|
out.flush()
|
|
os.fsync(out.fileno())
|
|
_replace(tmp, dest)
|
|
except BaseException:
|
|
Path(tmp).unlink(missing_ok=True)
|
|
raise
|
|
return size, sha.hexdigest()
|
|
|
|
|
|
def _copy_atomic(src: Path, dest: Path) -> tuple[int, str]:
|
|
with open(src, "rb") as handle:
|
|
return write_atomic(dest, iter(lambda: handle.read(1 << 20), b""))
|
|
|
|
|
|
def read_manifest(machine_dir: Path) -> Optional[dict]:
|
|
"""The manifest of the last run into `machine_dir`, or None when there is
|
|
none. Raises `BackupError` for one that cannot be read as one."""
|
|
path = machine_dir / MANIFEST_NAME
|
|
if not path.exists():
|
|
return None
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, ValueError) as exc:
|
|
raise BackupError(f"{path} cannot be read ({exc}) - nothing was written") from None
|
|
files = data.get("files") if isinstance(data, dict) else None
|
|
if not isinstance(files, list) or not all(
|
|
isinstance(entry, dict) and isinstance(entry.get("path"), str)
|
|
and safe_relative(entry["path"]) and isinstance(entry.get("size"), int)
|
|
and isinstance(entry.get("sha256"), str)
|
|
for entry in files
|
|
):
|
|
raise BackupError(f"{path} is not a backup manifest - nothing was written")
|
|
return data
|
|
|
|
|
|
@dataclass
|
|
class RunResult:
|
|
machine_dir: Path
|
|
written: list = field(default_factory=list)
|
|
unchanged: list = field(default_factory=list)
|
|
removed: list = field(default_factory=list)
|
|
skipped: list = field(default_factory=list)
|
|
|
|
def summary(self) -> str:
|
|
line = (f"Machine data backed up to {self.machine_dir}: {len(self.written)} written, "
|
|
f"{len(self.unchanged)} unchanged, {len(self.removed)} removed")
|
|
if self.skipped:
|
|
line += f"; skipped: {', '.join(self.skipped)}"
|
|
return line
|
|
|
|
|
|
def prune_empty(directory: Path, stop: Path) -> None:
|
|
while directory != stop and is_under(directory, stop):
|
|
try:
|
|
directory.rmdir()
|
|
except OSError:
|
|
return
|
|
directory = directory.parent
|
|
|
|
|
|
def _stack_version() -> Optional[str]:
|
|
from chemenu import version as version_mod
|
|
|
|
try:
|
|
return str(version_mod.read_version())
|
|
except version_mod.VersionError:
|
|
return None
|
|
|
|
|
|
def run(cfg: BackupConfig, root: "Path | str | None" = None) -> RunResult:
|
|
"""Mirror this machine's files into `<target>/<machine>/` and write its
|
|
manifest. See the module docstring for what it guarantees."""
|
|
root = Path(config.ROOT if root is None else root)
|
|
if not cfg.target.is_dir():
|
|
raise BackupError(
|
|
f"the backup target {cfg.target} is not a directory - nothing was written. Is the "
|
|
"synced folder there? `tools/wikitool backup init` creates the target"
|
|
)
|
|
machine_dir = cfg.machine_dir
|
|
if is_link(machine_dir) or (machine_dir.exists() and not machine_dir.is_dir()):
|
|
raise BackupError(f"{machine_dir} is not a plain directory - nothing was written")
|
|
previous = read_manifest(machine_dir)
|
|
if previous is None and machine_dir.is_dir() and any(machine_dir.iterdir()):
|
|
raise BackupError(
|
|
f"{machine_dir} is not empty and has no {MANIFEST_NAME} - it is not a folder this "
|
|
"backup wrote, so nothing was written. Set another backup.machine_name, or empty it"
|
|
)
|
|
|
|
collected = collect(cfg, root)
|
|
result = RunResult(machine_dir, skipped=list(collected.skipped))
|
|
before = {entry["path"]: entry for entry in (previous or {}).get("files", [])}
|
|
entries = []
|
|
machine_dir.mkdir(exist_ok=True)
|
|
for rel, source in sorted(collected.files.items()):
|
|
dest = _within(machine_dir, rel)
|
|
size, digest = file_digest(source)
|
|
old = before.get(rel)
|
|
if (old and old["size"] == size and old["sha256"] == digest and dest.is_file()
|
|
and not is_link(dest) and dest.stat().st_size == size):
|
|
result.unchanged.append(rel)
|
|
else:
|
|
size, digest = _copy_atomic(source, dest)
|
|
result.written.append(rel)
|
|
entries.append({"path": rel, "size": size, "sha256": digest})
|
|
|
|
for rel in sorted(set(before) - set(collected.files)):
|
|
dest = _within(machine_dir, rel)
|
|
if dest.is_file() and not is_link(dest):
|
|
dest.unlink()
|
|
result.removed.append(rel)
|
|
prune_empty(dest.parent, machine_dir)
|
|
|
|
manifest = {
|
|
"schema": MANIFEST_SCHEMA,
|
|
"created": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
"stack_version": _stack_version(),
|
|
"machine": cfg.machine_name,
|
|
"files": entries,
|
|
}
|
|
text = json.dumps(manifest, indent=2, ensure_ascii=False) + "\n"
|
|
write_atomic(cfg.manifest, [text.encode("utf-8")])
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Synced folders (`doctor`)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def sync_roots() -> list[Path]:
|
|
"""The folders a sync client keeps on this machine, as far as it says."""
|
|
roots = [Path(value) for value in (os.environ.get(var, "").strip() for var in SYNC_ENV_VARS) if value]
|
|
roots.append(Path.home().joinpath(*MAC_CLOUD_STORAGE))
|
|
return roots
|
|
|
|
|
|
def synced_folder_of(path: Path) -> Optional[Path]:
|
|
"""The synced folder `path` lies in, links and junctions resolved on both
|
|
sides, or None."""
|
|
real = real_path(path)
|
|
for candidate in sync_roots():
|
|
if is_under(real, real_path(candidate)):
|
|
return candidate
|
|
return None
|
|
|
|
|
|
def age_text(created: str, now: Optional[datetime] = None) -> str:
|
|
try:
|
|
then = datetime.fromisoformat(created)
|
|
except (TypeError, ValueError):
|
|
return "of unknown age"
|
|
seconds = ((now or datetime.now(timezone.utc)) - then).total_seconds()
|
|
for limit, unit in ((86400, "d"), (3600, "h"), (60, "min")):
|
|
if seconds >= limit:
|
|
return f"{int(seconds // limit)} {unit} old"
|
|
return "less than a minute old"
|