torben/chemenu
1
1 Fork 0
Code Issues 40 Pull Requests Actions Packages Releases 41 Wiki Activity

settings: the four per-feature files .wikitool-remotes/-telemetry/-upload/-tasks.json move into the three tiers and disappear; new command config migrate; 10.0.0-beta.1 (#208)
CI / verify (push) Failing after 7m41s
Details
CI / pwsh (push) Successful in 2m32s
Details
Release / release (push) Successful in 37s
Details
Release / wiki-mirror (push) Skipped
Details

Browse Source 
Files changed:
- .gitignore
- AGENTS.md
- CHANGES.md
- INSTALL.md
- VERSION
- docs/explanation/knowledge-and-commitment.md
- docs/explanation/telemetry-and-evaluation.md
- docs/explanation/why-gates-are-code.md
- docs/how-to/run-the-weekly-review.md
- docs/how-to/set-up-the-mcp-server.md
- docs/how-to/trace-and-score-sessions.md
- docs/how-to/troubleshoot.md
- docs/reference/commands/README.md
- docs/reference/commands/configuration.md
- docs/reference/commands/finding-and-checking.md
- docs/reference/commands/git.md
- docs/reference/commands/instance-health.md
- docs/reference/commands/pages.md
- docs/reference/commands/raw-material-and-uploads.md
- docs/reference/configuration.md
- docs/reference/exit-codes.md
- instructions/dev/doc-pull-through.md
- instructions/dev/tracker-testing.md
- instructions/gates.md
- instructions/ingest-queue.md
- instructions/mcp-read-server.md
- instructions/migrate-corpus.md
- instructions/migrations/10.0.0-settings-tiers.md
- instructions/setup-instance.md
- raw/CONTRACT.md
- reports/CONTRACT.md
- tools/CONTRACT.md
- tools/README.md
- tools/bugreport.py
- tools/chemenu/cli_contract.py
- tools/chemenu/commands/_util.py
- tools/chemenu/commands/config_cmd.py
- tools/chemenu/commands/docs_verify.py
- tools/chemenu/commands/doctor.py
- tools/chemenu/commands/export_cmd.py
- tools/chemenu/commands/git_publish.py
- tools/chemenu/commands/new_page.py
- tools/chemenu/commands/review_cmd.py
- tools/chemenu/commands/task_cmd.py
- tools/chemenu/commands/upload_cmd.py
- tools/chemenu/config.py
- tools/chemenu/config_shapes.py
- tools/chemenu/legacy_settings.py
- tools/chemenu/mcp/__main__.py
- tools/chemenu/mcp/server.py
- tools/chemenu/reference_doc.py
- tools/chemenu/review.py
- tools/chemenu/settings.py
- tools/chemenu/tasks/__init__.py
- tools/chemenu/tasks/caldav.py
- tools/chemenu/tasks/config.py
- tools/chemenu/tasks/superproductivity.py
- tools/chemenu/telemetry/policy.py
- tools/chemenu/tests/conftest.py
- tools/chemenu/tests/record_sp_fixtures.py
- tools/chemenu/tests/test_bugreport.py
- tools/chemenu/tests/test_caldav.py
- tools/chemenu/tests/test_config_shapes.py
- tools/chemenu/tests/test_doctor.py
- tools/chemenu/tests/test_git_publish.py
- tools/chemenu/tests/test_legacy_settings.py
- tools/chemenu/tests/test_mcp_server.py
- tools/chemenu/tests/test_new_page.py
- tools/chemenu/tests/test_review.py
- tools/chemenu/tests/test_settings.py
- tools/chemenu/tests/test_sp_recorded.py
- tools/chemenu/tests/test_superproductivity.py
- tools/chemenu/tests/test_task_cmd.py
- tools/chemenu/tests/test_tasks_config.py
- tools/chemenu/tests/test_telemetry_policy.py
- tools/chemenu/tests/test_tracker_live.py
- tools/chemenu/tests/test_upload.py
- tools/chemenu/tests/tracker_live.py
- tools/chemenu/upload.py

Co-Authored-By: Claude Code <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
This commit is contained in:
torbenandClaude Code committed 2026-10-11 06:54:35 +02:00
1 parent e085a8b6c1
commit db8f90c229
79 files changed
+2784 -1793
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/chemenu/cli_contract.py
+1
View File
@@ -317,6 +317,7 @@ GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = (
)),
("Configuration", (
"config set",
"config migrate",
)),
("Instance health", (
"doctor",
tools/chemenu/commands/_util.py
+30 -3
View File
@@ -1,6 +1,7 @@
"""Shared helpers for wikitool subcommands."""
from __future__ import annotations
import contextlib
import os
import re
import sys
@@ -95,18 +96,44 @@ def needs_clearance(msg: str) -> None:
raise typer.Exit(code=EXIT_NEEDS_CLEARANCE)
def load_settings():
def load_settings(only=None):
"""The tiered settings (`chemenu.settings.load`) for a command, or the
command's exit 1 naming file, key and tier."""
command's exit 1 naming file, key and tier. `only` names the sections the
command needs, so a broken section of another feature does not stop it."""
from chemenu import settings
from chemenu.errors import ValidationError
try:
return settings.load()
return settings.load(only=only)
except ValidationError as exc:
fail(str(exc))
def refuse_leftover(*features: str) -> None:
"""Exit 42 when a settings file of an earlier version still stands for one
of `features` (`chemenu.legacy_settings`): nothing reads it, and a human
runs `config migrate`."""
from chemenu import config, legacy_settings
try:
legacy_settings.refuse_leftover(config.ROOT, *features)
except legacy_settings.LegacySettingsLeftover as exc:
needs_clearance(str(exc))
@contextlib.contextmanager
def clearance_on_settings():
"""Exit 42 for what only a human can resolve while a command reads its
settings: a file of an earlier version still standing, or a secret source
that has to be unlocked first."""
from chemenu import legacy_settings, settings
try:
yield
except (legacy_settings.LegacySettingsLeftover, settings.SecretSourceLocked) as exc:
needs_clearance(str(exc))
def setting_value(section, name: str) -> Any:
"""One settings value for a command, a reference resolved: exit 1 when it
cannot be found, exit 42 when 1Password needs a human to unlock it first.
tools/chemenu/commands/config_cmd.py
+75 -5
View File
@@ -1,4 +1,4 @@
"""`wikitool config set` - writes one value of the tiered settings, checked.
"""`wikitool config set` and `config migrate` - write the tiered settings, checked.
The file a value lands in follows from its field's tier (`chemenu.settings`),
never from the caller: a plain `instance` value goes into `wikitool.json`, a
@@ -12,6 +12,10 @@ would see it, and an agent is what runs it most of the time - the value goes
into `WIKITOOL_SECRET_<NAME>`, the secrets file or 1Password, by the user. And a
field that names a program to run or affects a gate: like the Publish-Remote
Gate's allowlist, that one is changed by the user's own edit, never by an agent.
`config migrate` is the one-time move out of the four per-feature files of
earlier versions (`chemenu.legacy_settings`). It prints which key went where,
never a value.
"""
from __future__ import annotations
@@ -20,7 +24,7 @@ from typing import Any, Optional
import typer
from chemenu import cli_contract, config, settings
from chemenu import cli_contract, config, legacy_settings, settings
from chemenu.config_shapes import INSTANCE, SECRET, Field
from chemenu.commands._util import fail, needs_clearance, success
from chemenu.errors import ValidationError
@@ -98,9 +102,9 @@ def _plain_value(spec: Field, raw: str) -> Any:
),
),
examples=(
"tools/wikitool config set tracker.stale_days 14",
"tools/wikitool config set tracker.token --ref tracker_token",
"tools/wikitool config set tracker.url --ref op://Chemenu/tracker/url",
"tools/wikitool config set tasks.stalled_waiting_days 14",
"tools/wikitool config set caldav.app_password --ref caldav_app_password",
"tools/wikitool config set caldav.username --ref op://Chemenu/caldav/username",
),
never=(
"Never pass a secret's value on the command line - not to this command and not to any other.",
@@ -199,3 +203,69 @@ def set_command(
if created_id:
message += f" ({config.INSTANCE_SETTINGS_FILENAME} created with a new {settings.INSTANCE_ID_KEY})"
success(message)
@app.command("migrate")
@cli_contract.record(cli_contract.CommandRecord(
path="config migrate",
summary="Move the settings out of the per-feature files of earlier versions into the tiered settings.",
synopsis=(cli_contract.Variant(usage="config migrate"),),
properties=cli_contract.Properties(
effect=cli_contract.Effect.WRITE,
idempotent=cli_contract.Idempotent.YES,
atomic="Everything is computed and checked in memory first; the secrets file, "
"`wikitool.json` and `.wikitool-local.json` are each replaced in one rename, read back and "
"checked, and only then are the old files removed. Any failure before that puts every "
"target back to the bytes it had and leaves the old files as they were",
budget=cli_contract.Budget.COUNTED,
),
notes=(
"Moves the four per-feature settings files of earlier versions - telemetry, remotes, "
"upload and the task tracker, each named in the migration document - into the sections "
"`telemetry`, `remotes`, `upload`, and `tasks`, `superproductivity` and `caldav`: each value into the file its field's tier "
"names, a secret's value into the secrets file with only a reference in `wikitool.json`.",
"Every setting keeps its effective value, defaults included: the old content is read "
"with the same checks the new sections use, and a field the old file did not set is not "
"written.",
"Run once per checkout, since the old files and `.wikitool-local.json` belong to one "
"checkout. `.wikitool-tools.json` stays as it is.",
"Prints each key and the file it went to, plus anything the old file held that was not "
"read before and is not carried over - never a value.",
"A second run, or a run with no old file, prints that there is nothing to move and "
"changes no file.",
"A key the target files already hold with a different value stops the move before "
"anything is written: decide which one stands, remove the other, run again.",
),
failures=(
cli_contract.Failure(
cause="An old file does not pass the check the new section applies, a target key "
"holds a different value, or the files would not pass their check afterwards",
reaction="Nothing was changed. Fix the problem the message names, then re-run once",
),
),
examples=("tools/wikitool config migrate",),
never=(
"Never read the secrets file or an old file yourself to see a value.",
"Never delete an old file by hand to make a leftover message go away - the move is what "
"carries its settings over.",
),
see_also=(
"`wikitool doctor` - reports every leftover old file as a `FAIL`",
"`docs/reference/configuration.md` - the sections the files became",
"`instructions/migrations/10.0.0-settings-tiers.md` - the migration this command belongs to",
),
))
def migrate_command():
"""Move the per-feature settings files of earlier versions into the tiered settings."""
try:
result = legacy_settings.migrate(config.ROOT)
except ValidationError as exc:
fail(str(exc))
if not result.removed:
success("Nothing to migrate - no settings file of an earlier version is left")
return
for line in result.moved:
typer.echo(f" {line}")
for note in result.notes:
typer.echo(f" note: {note}")
success(f"Moved {len(result.moved)} setting(s); removed {', '.join(result.removed)}")
tools/chemenu/commands/docs_verify.py
+1 -8
View File
@@ -164,14 +164,7 @@ REQUIRED_IGNORE_CANARIES = (
# `incoming/` above: read by no command in the ordinary pipeline, not
# only uncommitted. No type subdirectory either, for the same reason.
"mcp-upload/probe.pdf",
# The `submit` tool's opt-in, same shape as `.wikitool-remotes.json`/
# `.wikitool-telemetry.json` a few lines below - per-checkout, never
# committed.
".wikitool-upload.json",
# The task-tracker provider opt-in (Gitea #124) - same shape again:
# per-checkout, never committed, once a credential lands in it.
".wikitool-tasks.json",
# The live suite's tracker profiles (Gitea #156) - credentials again, one file per tracker.
# The live suite's tracker profiles (Gitea #156) - references to credentials, one file per tracker.
".wikitool-tasks.d/probe.json",
# `raw capture`'s git cache (Gitea #177) - fetched repositories, recomputable,
# and in the way of `publish`'s `git add -A` like the coverage output above.
tools/chemenu/commands/doctor.py
+78 -46
View File
@@ -21,8 +21,9 @@ from typing import Optional
import typer
from rich.console import Console
from chemenu import cli_contract, config, config_shapes, conventions, filters, kb_collections, prerequisites, toolpaths, version as version_mod
from chemenu.commands import git_publish, instructions_cmd
from chemenu import cli_contract, config, config_shapes, conventions, filters, kb_collections, legacy_settings, prerequisites, toolpaths, version as version_mod
from chemenu.commands import instructions_cmd
from chemenu.errors import ChemenuError
from chemenu.commands._util import rel_path
from chemenu.session import ENV_VAR as SESSION_ENV_VAR
from chemenu.session import session_id_source as _session_id_source
@@ -422,13 +423,29 @@ def check_environment() -> Check:
return Check("environment", "OK", f"{config.ENVIRONMENT_FILE} present and filled")
def check_legacy_settings() -> list[Check]:
"""One FAIL per settings file of an earlier version that is still there.
Nothing reads those files any more, so a leftover is a setting that is not
in effect - for the Publish-Remote Gate and the upload limits that means a
safeguard silently off, which is why this is a FAIL and not a note."""
return [
Check(
"legacy-settings", "FAIL",
f"{name} is a settings file of an earlier version and no longer read",
legacy_settings.MIGRATE_COMMAND,
)
for name in legacy_settings.present(config.ROOT)
]
def check_publish_remotes() -> Check:
"""Whether the Publish-Remote Gate is armed in this checkout.
Absent is a legitimate state, not a fault: a checkout with a single remote
and nothing private in it has nothing to protect, and making the file
mandatory would turn a safeguard into paperwork. So this never FAILs - it
reports, the way `environment` does.
mandatory would turn a safeguard into paperwork. So an absent allowlist never
FAILs - it reports, the way `environment` does. An unreadable one does.
It does WARN for the case that actually bites: more than one remote
configured and no allowlist. That is the shape a private instance has once
@@ -441,12 +458,21 @@ def check_publish_remotes() -> Check:
that means about the gate is how a checkout ends up trusting a safeguard
that is not running - which is worse than having none.
"""
urls = git_publish.read_allowed_push_urls()
from chemenu import settings
try:
section = settings.load(only=(config_shapes.REMOTES.key,)).section(config_shapes.REMOTES.key)
urls = list(section["allowed_push_urls"]) if section.configured else None
except ChemenuError as exc:
return Check(
"publish-remotes", "FAIL", str(exc),
"Fix the settings as the message says - a broken allowlist is not 'no allowlist'",
)
if urls is not None:
return Check(
"publish-remotes", "OK",
f"Gate armed: {len(urls)} allowed push target(s) in "
f"{config.PUBLISH_REMOTES_FILENAME}",
f"Gate armed: {len(urls)} allowed push target(s) in remotes.allowed_push_urls "
f"({config.LOCAL_SETTINGS_FILENAME})",
)
result = _git(["remote"])
remotes = [r for r in result.stdout.split() if r] if result is not None else []
@@ -454,13 +480,13 @@ def check_publish_remotes() -> Check:
return Check(
"publish-remotes", "WARN",
f"Gate not armed: {len(remotes)} remotes ({', '.join(remotes)}) and no "
f"{config.PUBLISH_REMOTES_FILENAME} - every one of them is a legal publish target",
f"Create {config.PUBLISH_REMOTES_FILENAME} naming the push URL this checkout "
"may publish to - see instructions/gates.md",
"remotes.allowed_push_urls - every one of them is a legal publish target",
f"Put the push URL this checkout may publish to into {config.LOCAL_SETTINGS_FILENAME} "
"(section remotes, key allowed_push_urls) - see instructions/gates.md",
)
return Check(
"publish-remotes", "OK",
f"Gate not armed: no {config.PUBLISH_REMOTES_FILENAME} - any push target passes "
"Gate not armed: no remotes.allowed_push_urls - any push target passes "
"(1 remote configured, nothing to confuse it with)",
)
@@ -486,8 +512,8 @@ def check_telemetry() -> Check:
Never `FAIL`s, the same line `check_publish_remotes` and
`check_environment` draw: both an enabled and a disabled tree are
legitimate states, and a `FAIL` would make one of them mandatory by the
back door. It `WARN`s on a `.wikitool-telemetry.json` the policy does not
read as written, since the policy itself drops that in silence. The tree walk below is fine here - `doctor` is not a hot path,
back door. It `WARN`s on a `telemetry` section of `.wikitool-local.json` the policy
does not read as written, since the policy itself drops that in silence. The tree walk below is fine here - `doctor` is not a hot path,
unlike the `stat` `emit()` does on every append.
"""
from chemenu.telemetry import policy as telemetry_policy
@@ -512,29 +538,31 @@ def check_telemetry() -> Check:
if deviations:
return Check(
"telemetry", "WARN",
f"{detail}. {config.TELEMETRY_FILENAME} is not read as written: {'; '.join(deviations)}",
f"Correct {config.TELEMETRY_FILENAME} to the shape in "
"docs/reference/configuration.md#telemetry, or delete it to get the default",
f"{detail}. The telemetry section of {config.LOCAL_SETTINGS_FILENAME} is not read as "
f"written: {'; '.join(deviations)}",
"Correct the section to the shape in docs/reference/configuration.md#telemetry, "
"or remove it to get the default",
)
return Check("telemetry", "OK", detail)
def _telemetry_file_deviations() -> list[str]:
"""What the telemetry policy silently drops or reinterprets in this
checkout's `.wikitool-telemetry.json` - nothing when the file is absent.
"""What the telemetry policy silently drops or reinterprets in the
`telemetry` section of this checkout's `.wikitool-local.json` - nothing when
the file or the section is absent.
The policy has to stay silent (a hook handler reads it on every tool
call); this is where the silence becomes visible."""
path = config.ROOT / config.TELEMETRY_FILENAME
path = config.ROOT / config.LOCAL_SETTINGS_FILENAME
if not path.is_file():
return []
try:
data = _json.loads(path.read_text(encoding="utf-8"))
except (OSError, _json.JSONDecodeError) as exc:
return [f"unreadable ({exc}), so the whole file is ignored"]
if not isinstance(data, dict):
return ["not a JSON object, so the whole file is ignored"]
return config_shapes.deviations(config_shapes.TELEMETRY, data)
return [f"{config.LOCAL_SETTINGS_FILENAME} is unreadable ({exc}), so the section is ignored"]
if not isinstance(data, dict) or config_shapes.TELEMETRY.key not in data:
return []
return config_shapes.deviations(config_shapes.TELEMETRY, data[config_shapes.TELEMETRY.key])
def check_upload_intake() -> Check:
@@ -542,27 +570,26 @@ def check_upload_intake() -> Check:
its quarantine is.
Absent is the *safe* default here, unlike `check_publish_remotes`'s "any
push target passes" absence: no `.wikitool-upload.json` means the write
push target passes" absence: no `upload` section means the write
path does not exist at all, not that it is unrestricted - so this never
`FAIL`s on a missing file. It does `FAIL` on one that parses to something
`FAIL`s on a missing section. It does `FAIL` on one that parses to something
invalid, because a broken opt-in must not silently disable the very
limits it exists to enforce.
"""
from chemenu import upload as upload_module
from chemenu.errors import ValidationError
try:
cfg = upload_module.read_config(config.ROOT)
except ValidationError as exc:
except ChemenuError as exc:
return Check(
"upload-intake", "FAIL", str(exc),
f"Fix or delete {config.UPLOAD_CONFIG_FILENAME} - a broken one is not treated as "
"Fix the upload section as the message says - a broken one is not treated as "
"'no limits'",
)
if cfg is None:
return Check(
"upload-intake", "OK",
f"submit tool not registered - no {config.UPLOAD_CONFIG_FILENAME}",
"submit tool not registered - no upload section in the settings",
)
pending = upload_module.list_submissions(config.ROOT)
return Check(
@@ -600,16 +627,16 @@ def check_tasks_provider() -> Check:
try:
cfg = tasks_config.read_config(config.ROOT)
except ValidationError as exc:
except ChemenuError as exc:
return Check(
"tasks-provider", "FAIL", str(exc),
f"Fix or delete {config.tasks_config_label(config.ROOT)} - a broken one is not "
"treated as 'no tracker configured'",
"Fix the tasks, superproductivity and caldav sections as the message says - a "
"broken one is not treated as 'no tracker configured'",
)
if cfg is None:
return Check(
"tasks-provider", "OK",
f"No {config.TASKS_CONFIG_FILENAME} - no task tracker configured (the weekly "
"No tasks section in the settings - no task tracker configured (the weekly "
"review needs one, everything else does not)",
)
@@ -620,11 +647,11 @@ def check_tasks_provider() -> Check:
from chemenu.tasks import superproductivity as sp
try:
sp_cfg = sp.SuperProductivityConfig.from_dict(cfg.provider_config)
sp_cfg = sp.SuperProductivityConfig.from_section(cfg.provider_config)
except ValidationError as exc:
return Check(
"tasks-provider", "FAIL", str(exc),
f"Fix the 'superproductivity' section of {config.tasks_config_label(config.ROOT)}",
"Fix the 'superproductivity' section of the settings",
)
# Only the configured access path is a finding (Gitea #133) - the
# other one is not attempted at all, so it has nothing to report.
@@ -646,11 +673,11 @@ def check_tasks_provider() -> Check:
from chemenu.tasks import caldav as cd
try:
cd_cfg = cd.CalDAVConfig.from_dict(cfg.provider_config)
cd_cfg = cd.CalDAVConfig.from_section(cfg.provider_config)
except ValidationError as exc:
return Check(
"tasks-provider", "FAIL", str(exc),
f"Fix the 'caldav' section of {config.tasks_config_label(config.ROOT)}",
"Fix the 'caldav' section of the settings",
)
state = cd.probe(cd_cfg)
return Check("tasks-provider", "OK", f"caldav: {state}{via}")
@@ -873,6 +900,7 @@ def run_doctor() -> list[Check]:
check_personalization(),
check_conventions(),
check_environment(),
*check_legacy_settings(),
check_publish_remotes(),
check_upload_intake(),
check_tasks_provider(),
@@ -921,13 +949,17 @@ def run_doctor() -> list[Check]:
"tool-owned section headings - a `FAIL` on any of the three.",
"Environment note: `ENVIRONMENT.md` is optional, so absent is `OK`; a still-templated "
"one is a `WARN`.",
"MCP `submit` tool: whether `.wikitool-upload.json` is present, absent or malformed, "
"its limits, and how many submissions wait in `mcp-upload/`. Absent is `OK` and means "
"the write path does not exist at all; malformed is a `FAIL`.",
"Task tracker: `.wikitool-tasks.json` present, absent or malformed - absent is `OK` "
"Legacy settings (`legacy-settings`): one `FAIL` for each per-feature settings file of "
"an earlier version (telemetry, remotes, upload, task tracker) that is still there. Nothing reads them any more, so the settings they hold are not in effect; "
"the fix is `tools/wikitool config migrate`.",
"MCP `submit` tool: whether the `upload` section of the settings is present, absent or "
"malformed, its limits, and how many submissions wait in `mcp-upload/`. Absent is `OK` "
"and means the write path does not exist at all; malformed is a `FAIL`.",
"Task tracker: the `tasks` section present, absent or malformed - absent is `OK` "
"(no tracker configured), malformed is a `FAIL`. When `WIKITOOL_TASKS_CONFIG` is set, "
"that file is read instead and the finding names it; a set variable that names no file "
"is a `FAIL`, never `OK`.",
"the profile it names is read instead of the `tasks`, `superproductivity` and `caldav` "
"sections and the finding names it; a set variable that names no file is a `FAIL`, "
"never `OK`.",
"For a configured `superproductivity` provider, the configured `access` path's own "
"state: `access: \"api\"` reports whether its local REST API answers `GET /health` "
"with a ready renderer right now, `access: \"snapshot\"` whether a backup file is ready. The other access "
@@ -943,9 +975,9 @@ def run_doctor() -> list[Check]:
"out until a human unlocks it a `WARN`.",
"Session id source: `OK` for `WIKITOOL_SESSION_ID` or a registered harness variable, "
"`WARN` only for the bare parent-pid fallback.",
"Telemetry: on or off and why - installation-form default, `.wikitool-telemetry.json`, "
"or `WIKI_TRACE` - and the current session's count and byte total against both caps; "
"never a `FAIL`. A `WARN` when `.wikitool-telemetry.json` is not read as written - "
"Telemetry: on or off and why - installation-form default, the `telemetry` section of "
"`.wikitool-local.json`, or `WIKI_TRACE` - and the current session's count and byte total "
"against both caps; never a `FAIL`. A `WARN` when that section is not read as written - "
"unreadable JSON, not an object, `enabled` not a JSON boolean, a cap not a positive "
"integer, or an unknown key - naming the field; telemetry itself drops or reinterprets "
"such a value without a word, so this is where it shows.",
tools/chemenu/commands/export_cmd.py
+1 -1
View File
@@ -27,7 +27,7 @@ status and diff; the token digests exactly that set (URL, branch, old tip, new
blob), so a moved branch, a changed guideline or a new target asks again. The
Publish-Remote Gate is deliberately not applied: the targets are declared by
committed manifests already, and listing the same URLs in
`.wikitool-remotes.json` would be the second declaration again.
`remotes.allowed_push_urls` would be the second declaration again.
"""
from __future__ import annotations
tools/chemenu/commands/git_publish.py
+34 -39
View File
@@ -63,7 +63,7 @@ from typing import NamedTuple, Optional
import typer
from chemenu import cli_contract, config, config_shapes, toolpaths
from chemenu.commands._util import fail, needs_clearance, success
from chemenu.commands._util import fail, load_settings, needs_clearance, refuse_leftover, success
from chemenu.telemetry import emit
# "Farza's Rule" from the Mass-Update Gate concept: halt and request
@@ -104,34 +104,20 @@ def _run(args: list[str], input: Optional[str] = None,
# fetchable by SHA until someone expires the server's reflogs.
#
# Like the other token-cleared gates this refuses with exit 42 and has **no flag that
# opens it**. The way past it is to name the URL in the file, which is an edit
# the user makes deliberately rather than something an agent can decide mid-run.
# opens it**. The way past it is to name the URL in `remotes.allowed_push_urls`
# of `.wikitool-local.json`, which is an edit the user makes deliberately rather
# than something an agent can decide mid-run.
def read_allowed_push_urls() -> Optional[list[str]]:
"""The URLs this checkout permits `publish` to push to, or None when the
file is absent (unrestricted - see config.PUBLISH_REMOTES_FILENAME)."""
path = config.ROOT / config.PUBLISH_REMOTES_FILENAME
if not path.is_file():
`remotes` section is not set (unrestricted - see `config_shapes.REMOTES`).
A remotes file of an earlier version is exit 42, never "unrestricted"."""
refuse_leftover("remotes")
section = load_settings(only=(config_shapes.REMOTES.key,)).section(config_shapes.REMOTES.key)
if not section.configured:
return None
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
fail(
f"{config.PUBLISH_REMOTES_FILENAME} is unreadable ({exc}). It decides where "
"`publish` may push, so a broken one is not treated as 'no restriction' - "
"fix the file or delete it deliberately."
)
return None
shape = config_shapes.REMOTES
checked = config_shapes.check(shape, data)
if checked.problems:
fail(
f"{config.PUBLISH_REMOTES_FILENAME} has no usable allowed-URL list: {checked.message()}. "
f"Expected: {config_shapes.render_expected(shape)}"
)
return None
return checked.values["allowed_push_urls"]
return list(section["allowed_push_urls"])
def push_url_for(remote: str) -> Optional[str]:
@@ -153,7 +139,7 @@ def publish_remote_refusal(remote: str, branch: str) -> Optional[str]:
if url is None:
return (
f"Publish-Remote Gate: '{remote}' resolves to no push URL, so this publish "
f"cannot be checked against {config.PUBLISH_REMOTES_FILENAME}. Nothing was "
f"cannot be checked against remotes.allowed_push_urls in {config.LOCAL_SETTINGS_FILENAME}. Nothing was "
"committed or pushed."
)
if url in allowed:
@@ -168,8 +154,8 @@ def publish_remote_refusal(remote: str, branch: str) -> Optional[str]:
"fetchable by SHA even after a force-push, until the server's reflogs are expired.\n\n"
"THE USER CANNOT SEE THIS OUTPUT. It went to your context, not to their screen.\n"
"Show them the two lines above and stop. There is no flag that opens this gate: if "
f"the target really is right, the user adds its URL to {config.PUBLISH_REMOTES_FILENAME} "
"themselves. Do not edit that file to get past this."
f"the target really is right, the user adds its URL to remotes.allowed_push_urls in "
f"{config.LOCAL_SETTINGS_FILENAME} themselves. Do not edit that file to get past this."
)
@@ -1559,11 +1545,14 @@ def commit_regenerated(remote: str, branch: str) -> None:
"accounts for both, by reason.",
"The `--confirm` token covers each counted path, the blob id of its contents and the "
"publish target: a different file list or edited contents need a new clearance.",
"Publish-Remote Gate: when the checkout carries `.wikitool-remotes.json` and the push "
"URL of `--remote` is not listed in it, exits 42 before the reconcile fetches anything. "
"The URL is read with `git remote get-url --push`, so a repointed remote does not pass "
"on its name. An absent file means unrestricted; a malformed one is an error, not "
"permission.",
"Publish-Remote Gate: when `.wikitool-local.json` carries `remotes.allowed_push_urls` and "
"the push URL of `--remote` is not listed in it, exits 42 before the reconcile fetches "
"anything. The URL is read with `git remote get-url --push`, so a repointed remote does "
"not pass on its name. An absent section means unrestricted; a malformed one is an "
"error, not permission.",
"The remotes settings file of an earlier version is not read: while it exists this "
"exits 42 and names `tools/wikitool config migrate`, so a restriction is never lost "
"silently.",
"`--path` (repeatable) scopes the whole operation - gate count, staging and commit - "
"to that subtree, with one exception: uncommitted generated files outside `--path` "
"(`kb/index.md`, `kb/log.md`, `kb/provenance.md`, every `INDEX.md` under `kb/`) are "
@@ -1614,10 +1603,16 @@ def commit_regenerated(remote: str, branch: str) -> None:
"from the gate's own refusal output",
),
cli_contract.Failure(
cause="`.wikitool-remotes.json` is unreadable or has no usable "
"`allowed_push_urls` list",
reaction="Show the error to the user and stop - a malformed file is not "
"permission, and fixing or deleting it is theirs to do",
cause="The settings files do not pass their check, or `remotes.allowed_push_urls` is "
"not a list of strings",
reaction="Show the error to the user and stop - a malformed section is not "
"permission, and fixing it is theirs to do",
),
cli_contract.Failure(
cause="The remotes settings file of an earlier version is still there",
reaction="Show the output to the user verbatim and stop; `tools/wikitool config "
"migrate` moves it, and the user decides when",
code=42,
),
cli_contract.Failure(
cause="Mass-Update Gate: `--threshold` (default 10) or more counted files would be "
@@ -1633,11 +1628,11 @@ def commit_regenerated(remote: str, branch: str) -> None:
code=42,
),
cli_contract.Failure(
cause="Publish-Remote Gate: `.wikitool-remotes.json` exists and the push URL of "
cause="Publish-Remote Gate: `remotes.allowed_push_urls` is set and the push URL of "
"`--remote` is not listed in it, or `--remote` resolves to no push URL",
reaction="Show the user the push URL it names and the allowed ones, and stop. This "
"gate has no token and no flag: only the user resolves it, by adding the URL to "
"that file",
"`.wikitool-local.json`",
code=42,
),
),
@@ -1653,7 +1648,7 @@ def commit_regenerated(remote: str, branch: str) -> None:
"`--force-with-lease`).",
"Never pass a `--confirm` or `--confirm-rebase` token the user has not seen and "
"approved.",
"Never edit `.wikitool-remotes.json` to get past a Publish-Remote refusal - that is "
"Never edit `remotes.allowed_push_urls` to get past a Publish-Remote refusal - that is "
"opening a gate on your own initiative.",
),
see_also=(
tools/chemenu/commands/new_page.py
+18 -13
View File
@@ -37,6 +37,7 @@ from chemenu.commands._util import (
check_path_budget,
check_target_free,
check_title,
clearance_on_settings,
fail,
needs_clearance,
parse_set_fields,
@@ -292,19 +293,23 @@ def _ensure_tracker_project(page_title: str, *, resume: bool) -> Optional[str]:
exact name, and the caller's next step is the only one left: write the
page.
"""
cfg = tasks_config.read_config(config.ROOT)
if cfg is None:
return None
with clearance_on_settings():
try:
cfg = tasks_config.read_config(config.ROOT)
except ValidationError as exc:
fail(str(exc))
if cfg is None:
return None
reader = tasks.build_reader(cfg)
try:
writer = tasks.build_writer(cfg, reader)
except ValidationError as exc:
# E.g. a superproductivity instance with access: "snapshot" (Gitea
# #133) - the tracker is read-only from here, so this refuses before
# either the collision check or the page write, exactly like any
# other precondition failure.
fail(str(exc))
reader = tasks.build_reader(cfg)
try:
writer = tasks.build_writer(cfg, reader)
except ValidationError as exc:
# E.g. a superproductivity instance with access: "snapshot" (Gitea
# #133) - the tracker is read-only from here, so this refuses before
# either the collision check or the page write, exactly like any
# other precondition failure.
fail(str(exc))
existing = find_project(reader, page_title)
if existing is not None:
@@ -407,7 +412,7 @@ def _ensure_tracker_project(page_title: str, *, resume: bool) -> Optional[str]:
"passed with `--set`; `new` never guesses it and refuses `unknown` for it.",
"Produces structurally correct frontmatter and a body skeleton only - the prose "
"(Description, Summary, judgment calls about relationships) is written afterwards.",
"`new project`: with `.wikitool-tasks.json` configuring a task tracker, also makes sure "
"`new project`: with the `tasks` section of the settings configuring a task tracker, also makes sure "
"a same-named tracker project exists - one name, one identity. No tracker configured is "
"a legitimate, explicitly announced state: page only.",
"`new project`: tracker before page. The tracker side is settled first, so a failure "
tools/chemenu/commands/review_cmd.py
+24 -14
View File
@@ -11,7 +11,7 @@ import json
import typer
from chemenu import cli_contract, config
from chemenu.commands._util import fail
from chemenu.commands._util import clearance_on_settings, fail
from chemenu.errors import ValidationError
from chemenu.review import ALL_CHECKS, ReviewReport, run_review
@@ -105,10 +105,12 @@ def report_to_dict(report: ReviewReport) -> dict:
"tracker project with no determinable creation date - is its own finding "
"(`waiting_no_follow_up`/`project_age_unknown`) rather than a silent skip.",
"Thresholds come from the tracker configuration, never from the schema.",
"The tracker configuration is read from `.wikitool-tasks.json`, or from the file "
"`WIKITOOL_TASKS_CONFIG` names when that variable is set (so one checkout can be "
"run against several trackers in turn). A set variable that names no file is an "
"error, never \"no tracker configured\".",
"The tracker configuration is read from the `tasks`, `superproductivity` and `caldav` "
"sections of the settings, or from the profile `WIKITOOL_TASKS_CONFIG` names when that "
"variable is set (so one checkout can be run against several trackers in turn). A set "
"variable that names no file is an error, never \"no tracker configured\".",
"The tracker settings file of an earlier version exits 42 and names `tools/wikitool "
"config migrate`; so does a secret source that needs unlocking.",
"Text output is one `[check] project: message` line per finding, preceded by a "
"`Source:` line naming which access path answered and, for `superproductivity`'s "
"`access: \"snapshot\"`, the snapshot's age. `--json` carries the same findings plus "
@@ -122,13 +124,20 @@ def report_to_dict(report: ReviewReport) -> dict:
),
failures=(
cli_contract.Failure(
cause="No `.wikitool-tasks.json`, or a malformed one - a clear \"no tracker "
cause="No `tasks` section in the settings, or a malformed one - a clear \"no tracker "
"configured\" message",
reaction="Not fixed by retrying unchanged - configure or repair "
"`.wikitool-tasks.json` first",
reaction="Not fixed by retrying unchanged - configure or repair the `tasks`, "
"`superproductivity` and `caldav` sections first",
),
cli_contract.Failure(
cause="`WIKITOOL_TASKS_CONFIG` names a file that does not exist or is broken",
cause="The tracker settings file of an earlier version is still there, or a secret "
"source (1Password) needs unlocking",
reaction="Show the output to the user verbatim and stop - `tools/wikitool config "
"migrate` or the unlock is theirs to run",
code=42,
),
cli_contract.Failure(
cause="`WIKITOOL_TASKS_CONFIG` names a profile that does not exist or is broken",
reaction="Not transient - fix the path or unset the variable",
),
cli_contract.Failure(
@@ -159,11 +168,12 @@ def review_command(
report the five staleness/mismatch checks (#119 D10/D26). Read-only -
stores nothing, not even a reports/ file (#119 D3), and is exempt from
the Iteration Budget Gate like `search`."""
try:
report = run_review(config.ROOT)
except ValidationError as exc:
fail(str(exc))
return
with clearance_on_settings():
try:
report = run_review(config.ROOT)
except ValidationError as exc:
fail(str(exc))
return
if json_out:
typer.echo(json.dumps(report_to_dict(report), indent=2))
tools/chemenu/commands/task_cmd.py
+68 -42
View File
@@ -17,11 +17,12 @@ transaction, because nothing here shares state with the page-creation path
the way `new project`'s own tracker-then-page order does within a single
command.
Neither write ever exits 42, unlike `new project`: every provider that
offers a write path at all has a real per-item call (Super Productivity's
`POST /tasks`, where `POST /projects` does not exist), so there is no
human-clearance step to wait on - a missing project or tag is an ordinary
refusal. `task close` takes the provider's own id, never a title: an item's
Neither write waits for clearance on the tracker, unlike `new project`: every
provider that offers a write path at all has a real per-item call (Super
Productivity's `POST /tasks`, where `POST /projects` does not exist) - a missing
project or tag is an ordinary refusal. The one exit 42 is about the settings, not the
tracker: the tracker settings file of an earlier version still standing, or a secret
source that needs unlocking. `task close` takes the provider's own id, never a title: an item's
tracker-side identity is opaque, while a project's *name* is the one coupling
`kb/` and the tracker share. `--notes` is stored verbatim and never parsed -
the same posture a WAITING item's own title has for the person named in it.
@@ -41,7 +42,7 @@ from typing import Optional
import typer
from chemenu import cli_contract, config, tasks
from chemenu.commands._util import fail, success
from chemenu.commands._util import clearance_on_settings, fail, success
from chemenu.errors import ValidationError
from chemenu.tasks import config as tasks_config
@@ -59,10 +60,11 @@ def _parse_follow_up_at(text: str) -> datetime.date:
def _read_config():
try:
return tasks_config.read_config(config.ROOT)
except ValidationError as exc:
fail(str(exc))
with clearance_on_settings():
try:
return tasks_config.read_config(config.ROOT)
except ValidationError as exc:
fail(str(exc))
@app.command("new")
@@ -94,15 +96,15 @@ def _read_config():
"`--follow-up-at` is refused without `--waiting` - it is never a due date on its own.",
"`--notes` carries a freetext backref (e.g. to the `kb/` source page the item came "
"from), stored verbatim, never parsed.",
"The tracker configuration is read from `.wikitool-tasks.json`, or from the file "
"`WIKITOOL_TASKS_CONFIG` names when that variable is set (so one checkout can be "
"run against several trackers in turn). A set variable that names no file is an "
"error, never \"no tracker configured\".",
"No `.wikitool-tasks.json` fails immediately with a \"no tracker configured\" message.",
"The tracker configuration is read from the `tasks`, `superproductivity` and `caldav` "
"sections of the settings, or from the profile `WIKITOOL_TASKS_CONFIG` names when that "
"variable is set (so one checkout can be run against several trackers in turn). A set "
"variable that names no file is an error, never \"no tracker configured\".",
"No `tasks` section fails immediately with a \"no tracker configured\" message.",
"A provider whose configured access path has no write path (Super Productivity's "
"`access: \"snapshot\"`) refuses entirely with exit 1, naming the `access: \"api\"` "
"instance to use instead.",
"**Never exits 42.** A `--project` matching no tracker project, or `--waiting` against "
"**Exits 42 only for the settings** (the tracker settings file of an earlier version, a locked secret source). A `--project` matching no tracker project, or `--waiting` against "
"a provider that cannot represent it right now (Super Productivity: the `waiting` tag "
"does not exist yet, and its API cannot create tags), are ordinary exit-1 refusals "
"that create nothing.",
@@ -110,11 +112,18 @@ def _read_config():
),
failures=(
cli_contract.Failure(
cause="No `.wikitool-tasks.json` - no tracker configured",
cause="No `tasks` section in the settings - no tracker configured",
reaction="Not transient - configure a tracker first",
),
cli_contract.Failure(
cause="`WIKITOOL_TASKS_CONFIG` names a file that does not exist or is broken",
cause="The tracker settings file of an earlier version is still there, or a secret "
"source (1Password) needs unlocking",
reaction="Show the output to the user verbatim and stop - `tools/wikitool config "
"migrate` or the unlock is theirs to run",
code=42,
),
cli_contract.Failure(
cause="`WIKITOOL_TASKS_CONFIG` names a profile that does not exist or is broken",
reaction="Not transient - fix the path or unset the variable",
),
cli_contract.Failure(
@@ -209,15 +218,16 @@ def task_new_command(
cfg = _read_config()
if cfg is None:
fail(
f"No {config.TASKS_CONFIG_FILENAME} - no task tracker is configured, so there is "
"No tasks section in the settings - no task tracker is configured, so there is "
"nowhere to create this item. Configure one first."
)
reader = tasks.build_reader(cfg)
try:
writer = tasks.build_writer(cfg, reader)
except ValidationError as exc:
fail(str(exc))
with clearance_on_settings():
reader = tasks.build_reader(cfg)
try:
writer = tasks.build_writer(cfg, reader)
except ValidationError as exc:
fail(str(exc))
try:
writer.create_item(
@@ -264,11 +274,18 @@ def task_new_command(
code=0,
),
cli_contract.Failure(
cause="No `.wikitool-tasks.json` - no tracker configured",
cause="No `tasks` section in the settings - no tracker configured",
reaction="Not transient - configure a tracker first, then retry once",
),
cli_contract.Failure(
cause="`WIKITOOL_TASKS_CONFIG` names a file that does not exist or is broken",
cause="The tracker settings file of an earlier version is still there, or a secret "
"source (1Password) needs unlocking",
reaction="Show the output to the user verbatim and stop - `tools/wikitool config "
"migrate` or the unlock is theirs to run",
code=42,
),
cli_contract.Failure(
cause="`WIKITOOL_TASKS_CONFIG` names a profile that does not exist or is broken",
reaction="Not transient - fix the path or unset the variable",
),
),
@@ -293,11 +310,12 @@ def task_list_command(
cfg = _read_config()
if cfg is None:
fail(
f"No {config.TASKS_CONFIG_FILENAME} - no task tracker is configured, so there is "
"No tasks section in the settings - no task tracker is configured, so there is "
"nothing to list."
)
reader = tasks.build_reader(cfg)
with clearance_on_settings():
reader = tasks.build_reader(cfg)
try:
items = reader.open_items(project).items
except ValidationError as exc:
@@ -330,23 +348,30 @@ def task_list_command(
"write this stack makes.",
"`--id` is the provider's own item id, from `task list` or a `review` finding - never a "
"title.",
"The tracker configuration is read from `.wikitool-tasks.json`, or from the file "
"`WIKITOOL_TASKS_CONFIG` names when that variable is set (so one checkout can be "
"run against several trackers in turn). A set variable that names no file is an "
"error, never \"no tracker configured\".",
"No `.wikitool-tasks.json` fails with a \"no tracker configured\" message.",
"The tracker configuration is read from the `tasks`, `superproductivity` and `caldav` "
"sections of the settings, or from the profile `WIKITOOL_TASKS_CONFIG` names when that "
"variable is set (so one checkout can be run against several trackers in turn). A set "
"variable that names no file is an error, never \"no tracker configured\".",
"No `tasks` section fails with a \"no tracker configured\" message.",
"A provider whose configured access path has no write path (Super Productivity's "
"`access: \"snapshot\"`) refuses entirely with exit 1, naming the `access: \"api\"` "
"instance to use instead.",
"**Never exits 42.**",
"**Exits 42 only for the settings** (the tracker settings file of an earlier version, a locked secret source).",
),
failures=(
cli_contract.Failure(
cause="No `.wikitool-tasks.json` - no tracker configured",
cause="No `tasks` section in the settings - no tracker configured",
reaction="Not transient - configure a tracker first",
),
cli_contract.Failure(
cause="`WIKITOOL_TASKS_CONFIG` names a file that does not exist or is broken",
cause="The tracker settings file of an earlier version is still there, or a secret "
"source (1Password) needs unlocking",
reaction="Show the output to the user verbatim and stop - `tools/wikitool config "
"migrate` or the unlock is theirs to run",
code=42,
),
cli_contract.Failure(
cause="`WIKITOOL_TASKS_CONFIG` names a profile that does not exist or is broken",
reaction="Not transient - fix the path or unset the variable",
),
cli_contract.Failure(
@@ -385,15 +410,16 @@ def task_close_command(
cfg = _read_config()
if cfg is None:
fail(
f"No {config.TASKS_CONFIG_FILENAME} - no task tracker is configured, so there is "
"No tasks section in the settings - no task tracker is configured, so there is "
"nothing to close."
)
reader = tasks.build_reader(cfg)
try:
writer = tasks.build_writer(cfg, reader)
except ValidationError as exc:
fail(str(exc))
with clearance_on_settings():
reader = tasks.build_reader(cfg)
try:
writer = tasks.build_writer(cfg, reader)
except ValidationError as exc:
fail(str(exc))
try:
writer.close_item(item_id)
tools/chemenu/commands/upload_cmd.py
+1 -1
View File
@@ -46,7 +46,7 @@ def _call(fn, *args, **kwargs):
notes=(
"Lists every submission waiting in `mcp-upload/`, oldest id first: id, filename, size, "
"submitter.",
"Only ever non-empty when `.wikitool-upload.json` opts the checkout into the MCP "
"Only ever non-empty when the `upload` section of `.wikitool-local.json` opts the checkout into the MCP "
"server's `submit` tool.",
"A submission directory with a corrupt manifest is silently skipped.",
"Never fails; read-only and safe to retry freely.",
tools/chemenu/config.py
+7 -66
View File
@@ -221,56 +221,13 @@ LICENSE_FILES = ("LICENSE", "LICENSE-CONTENT", "NOTICE")
# rather than defining its own, per AGENTS.md invariant 8.
RELEASE_STAMP_FILENAME = ".wikitool-release.json"
# Per-checkout telemetry opt-in/opt-out plus its two quantity caps (see
# `telemetry/policy.py`). Same posture as `PUBLISH_REMOTES_FILENAME` below: it
# answers a question about *this* checkout, so it is per-checkout and
# gitignored, ships no `.template`, and its absence is a legitimate state -
# the installation-form default (keyed off `RELEASE_STAMP_FILENAME`) applies.
# `instructions/setup-instance.md`'s Telemetry decision point writes it from
# the operator's answer. Its fields are the record `config_shapes.TELEMETRY`,
# as for the three files below - each has exactly one.
TELEMETRY_FILENAME = ".wikitool-telemetry.json"
# Which push targets `publish` may write to, for a checkout that says so
# (fields: `config_shapes.REMOTES`). The
# danger this addresses is one checkout's content reaching another checkout's
# remote - a private instance pushing its own `kb/` to a public repository,
# where it cannot be taken back.
#
# It pins **URLs, not remote names**: a name-based list would pass a `publish`
# whose `origin` had been repointed, which is the failure it exists to catch.
#
# Per-checkout and gitignored, like `ENVIRONMENT.md` and for the same reason:
# two clones of this repo push to two different places, so a committed copy
# would hand the second one an answer that is wrong rather than missing. Absent
# means unrestricted - `doctor` reports it, and the Publish-Remote Gate simply
# does not apply. A checkout that holds private content should have one; see
# instructions/gates.md.
PUBLISH_REMOTES_FILENAME = ".wikitool-remotes.json"
# Opt-in for the MCP server's `submit` tool (Gitea #32); its fields are the
# record `config_shapes.UPLOAD`. Same posture as the two
# above - per-checkout, gitignored, no `.template` - but its absence means
# something stronger than "unrestricted": **the write path does not exist at
# all**, the tool is not registered. The safe direction, and a structural
# opt-in rather than a flag - see `chemenu.upload.read_config`.
UPLOAD_CONFIG_FILENAME = ".wikitool-upload.json"
# The task-tracker provider opt-in (Gitea #124, D25/D30): which provider this
# instance's GTD review reads/writes through, its connection details, and the
# review's three staleness thresholds - fields in `config_shapes.TASKS`. Same
# posture as the three files above -
# per-checkout, gitignored once a credential lands in it, no `.template` - and
# its absence is a legitimate state, the same posture `UPLOAD_CONFIG_FILENAME`
# takes: an instance with no tracker configured runs `doctor` and everything
# else just fine, it only can't run the weekly review (#125, not yet built).
TASKS_CONFIG_FILENAME = ".wikitool-tasks.json"
# Reads a tracker configuration from another file than `<root>/.wikitool-tasks.json`
# (Gitea #156), so one checkout can be run against several trackers one after the
# other without swapping a file. Registered in `_WIKITOOL_ENV`
# (tools/chemenu/tests/conftest.py). A relative value is taken from the working
# directory. Set but pointing at nothing is an error, never "no tracker".
# Names a profile - a file in the format of `.wikitool-local.json` holding only
# the `tasks`, `superproductivity` and `caldav` sections - that replaces those
# three sections from both settings files, so one checkout can be run against
# several trackers one after the other without swapping a file. Registered in
# `_WIKITOOL_ENV` (tools/chemenu/tests/conftest.py). A relative value is taken
# from the working directory. Set but pointing at nothing is an error, never
# "no tracker".
ENV_TASKS_CONFIG = "WIKITOOL_TASKS_CONFIG"
@@ -317,22 +274,6 @@ def tasks_config_override() -> "Path | None":
return Path(value).expanduser().resolve() if value else None
def tasks_config_path(root: "Path | str") -> Path:
"""The tracker configuration file this process reads: the override when set,
else `<root>/.wikitool-tasks.json`."""
override = tasks_config_override()
return override if override is not None else Path(root) / TASKS_CONFIG_FILENAME
def tasks_config_label(root: "Path | str") -> str:
"""How a message names the file `tasks_config_path` reads - the bare file
name by default, the actual path and the variable when the override is on."""
override = tasks_config_override()
if override is None:
return TASKS_CONFIG_FILENAME
return f"{override} (from {ENV_TASKS_CONFIG})"
def default_author() -> str | None:
"""The author to stamp a new source page with, per instance.
tools/chemenu/config_shapes.py
+189 -316
View File
@@ -1,30 +1,24 @@
"""The shapes of the per-checkout configuration files an operator writes by hand.
"""The shapes of the tiered settings sections.
Four files - `.wikitool-telemetry.json`, `.wikitool-remotes.json`,
`.wikitool-upload.json` and `.wikitool-tasks.json` (names in `chemenu.config`)
- each have exactly one record here, and that record is the only place their
shape is written down. The loader that reads a file takes its field names,
required fields, defaults, coercions, conditions and the `Expected:` text of its
error messages from the record; `docs reference --apply` renders the same record
into `docs/reference/configuration.md`. A second description of a shape, in a
loader or in prose, is what used to drift.
`wikitool.json` and `.wikitool-local.json` (`chemenu.settings`) hold one section
per feature; each section has exactly one record here, and that record is the
only place its shape is written down. The loader that reads a section takes its
field names, required fields, defaults, coercions and the tier of each field
from the record; `docs reference --apply` renders the same record into
`docs/reference/configuration.md`. A second description of a shape, in a loader
or in prose, is what used to drift.
The records describe what the loaders **accept today**, leniency included:
`int()` on a number given as a string, `bool()` on `enabled`, unknown top-level
keys ignored. Tightening any of that would reject a file an instance already
has, which is a different kind of change than writing the shape down.
`check()` never raises. Each loader keeps its own posture towards a problem -
`publish` stops through `fail()`, the upload and tracker loaders raise
`ValidationError`, and telemetry drops what it cannot use in silence, because a
hook handler reads it on every tool call. `deviations()` is the strict reading
`doctor` uses to report what that silence would otherwise hide.
The same records describe the sections of the two tiered settings files,
`wikitool.json` and `.wikitool-local.json` (`SETTINGS_SECTIONS`, read by
`chemenu.settings`). There every field also declares the tier it belongs to,
Every field declares the tier it belongs to (`INSTANCE`, `CHECKOUT`, `SECRET`),
and the rule that a field naming a program or affecting a gate is never
`instance` is checked when the `Field` is declared (`Field.__post_init__`).
`INSTANCE` is checked when the `Field` is declared (`Field.__post_init__`).
The records describe what the loaders **accept**, leniency included: `int()` on
a number given as a string, `bool()` on `enabled`. `check()` never raises. Each
loader keeps its own posture towards a problem - `publish` stops through
`fail()`, the upload and tracker loaders raise `ValidationError`, and telemetry
drops what it cannot use in silence, because a hook handler reads it on every
tool call. `deviations()` is the strict reading `doctor` uses to report what
that silence would otherwise hide.
The three machine-written files beside them - `.wikitool-release.json`,
`.wikitool-kb.json`, `.wikitool-tools.json` - are deliberately not here: nobody
@@ -42,12 +36,7 @@ from typing import Any, Callable, Optional
from chemenu import config
IGNORE = "ignore"
REJECT = "reject"
# The three tiers a settings field can belong to (`chemenu.settings`). A field
# of one of the four per-feature files above carries none: those files predate
# the tiers and are read by their own loaders.
# The three tiers a settings field can belong to (`chemenu.settings`).
INSTANCE = "instance"
CHECKOUT = "checkout"
SECRET = "secret"
@@ -70,9 +59,9 @@ class Field:
applies to the value it finds - it may accept more than `json_type`, which
is the leniency `deviations()` reports - and raises `TypeError`/`ValueError`
on a value it cannot use. `check` then judges the coerced value and returns
a problem, or None. A field with `fields` is a nested object.
a problem, or None.
The last four attributes matter only to a settings section
The last four attributes are the field's place in the tiered settings
(`chemenu.settings`). `tier` says which file the field's value lives in.
`private` marks private infrastructure - a host name, an internal URL - that
may sit in the committed `wikitool.json` only as a reference, never as
@@ -95,7 +84,6 @@ class Field:
empty_is_missing: bool = False
required_why: str = ""
lenient: str = ""
fields: tuple["Field", ...] = ()
tier: str = ""
private: bool = False
names_command: bool = False
@@ -118,8 +106,6 @@ class Field:
f"{where}: a private field is tier {CHECKOUT!r} - its plain value never goes "
"into the committed file, only a reference to it may"
)
if self.tier and self.fields:
raise ValueError(f"{where}: a settings field is a single value, not a nested object")
@property
def user_only(self) -> bool:
@@ -135,13 +121,10 @@ class Field:
@dataclass(frozen=True)
class Shape:
"""One configuration object: a whole file, or a provider's section of one.
`key` is the file name for a file and the section's key for a section.
`discriminator` names a field whose value picks one of `variants` - an
extra set of fields valid only alongside that value. `section_key` names a
field whose value picks one of `sections`, a nested object stored under the
chosen value's own key."""
"""One settings section: `key` is its key in `wikitool.json` and
`.wikitool-local.json`, `fields` are its fields, each with its tier.
`absent` and `malformed` say what a missing section and a broken one mean
to the feature that reads it."""
key: str
title: str
@@ -150,33 +133,13 @@ class Shape:
examples: tuple[dict, ...]
absent: str = ""
malformed: str = ""
unknown_keys: str = IGNORE
unknown_text: str = ""
unknown_why: str = ""
discriminator: str = ""
variants: tuple[tuple[str, tuple[Field, ...]], ...] = ()
section_key: str = ""
sections: tuple["Shape", ...] = ()
def field(self, name: str) -> Field:
for candidate in self.fields + tuple(f for _, fs in self.variants for f in fs):
for candidate in self.fields:
if candidate.name == name:
return candidate
raise KeyError(name)
def variant_fields(self, value: Any) -> Optional[tuple[Field, ...]]:
return dict(self.variants).get(value)
def choices(self, field: Field) -> tuple[str, ...]:
"""The values `field` may take: its own list, or - for the
discriminator and the section key - the variants and sections that
exist, so a new one cannot be added without the choice following."""
if field.name == self.discriminator:
return tuple(value for value, _ in self.variants)
if field.name == self.section_key:
return tuple(s.key for s in self.sections)
return field.choices
@dataclass(frozen=True)
class Problem:
@@ -206,19 +169,21 @@ class Checked:
# ---------------------------------------------------------------------------
def _exact_str(value: Any) -> str:
def exact_str(value: Any) -> str:
if not isinstance(value, str):
raise TypeError("not a string")
return value
def _str_list(value: Any) -> list[str]:
def str_list(value: Any) -> list[str]:
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
raise TypeError("not a list of strings")
return list(value)
def _extensions(value: Any) -> tuple[str, ...]:
def extensions(value: Any) -> tuple[str, ...]:
if not isinstance(value, list):
raise TypeError("not a list")
return tuple(sorted({str(ext).lower() for ext in value}))
@@ -226,163 +191,180 @@ def _positive(value: int) -> Optional[str]:
return None if value > 0 else "must be positive"
def _dotted_extensions(value: tuple[str, ...]) -> Optional[str]:
def dotted_extensions(value: tuple[str, ...]) -> Optional[str]:
if not value:
return "must not be empty"
not_dotted = [ext for ext in value if not ext.startswith(".")]
return f"must each start with '.': {not_dotted}" if not_dotted else None
def _positive_integer(name: str, description: str, **kwargs: Any) -> Field:
def positive_integer(name: str, description: str, **kwargs: Any) -> Field:
return Field(
name, "positive integer", description, json_type="integer", coerce=int,
check=_positive, lenient="A number written as a string is accepted.", **kwargs,
)
_SCHEMA = Field(
"schema", "any", "Accepted and not read. Conventionally `1`; no version of this file is "
"told apart by it yet.",
)
def _required_string(name: str, description: str, **kwargs: Any) -> Field:
return Field(name, "string", description, required=True, json_type="string", coerce=exact_str,
empty_is_missing=True, **kwargs)
# ---------------------------------------------------------------------------
# The records
# The sections
# ---------------------------------------------------------------------------
PROVIDERS = ("superproductivity", "caldav")
SUPERPRODUCTIVITY_ACCESS = ("api", "snapshot")
TELEMETRY = Shape(
key=config.TELEMETRY_FILENAME,
key="telemetry",
title="Telemetry",
purpose="Turns telemetry on or off for this checkout, against the installation-form default, "
"and sets its two quantity caps.",
fields=(
_SCHEMA,
Field(
"enabled", "boolean", "Telemetry on or off. `WIKI_TRACE` beats it in both directions.",
default_text="the installation-form default", json_type="boolean", coerce=bool,
lenient="Any other value is read by its truthiness, so `\"no\"` means on.",
tier=CHECKOUT,
),
_positive_integer(
positive_integer(
"max_session_bytes", "Byte cap per session trace. `WIKI_TRACE_MAX_SESSION_BYTES` beats it.",
default=5 * 1024 * 1024,
default=5 * 1024 * 1024, tier=CHECKOUT,
),
_positive_integer(
positive_integer(
"keep_sessions", "How many session directories retention keeps. "
"`WIKI_TRACE_KEEP_SESSIONS` beats it.", default=250,
"`WIKI_TRACE_KEEP_SESSIONS` beats it.", default=250, tier=CHECKOUT,
),
),
examples=({"enabled": True, "max_session_bytes": 5242880, "keep_sessions": 250},),
absent="The installation-form default applies: on in a clone of the origin repository, off in "
"an instance.",
malformed="Never an error. Unreadable JSON or a non-object counts as an absent file, and a cap "
"that is not a positive integer as an absent cap. `doctor` reports each such deviation as a "
"`WARN` under `telemetry`, naming the field.",
unknown_text="Ignored; `doctor` reports them as a `WARN`.",
malformed="The hook that reads it never fails on it: a value it cannot use counts as absent, "
"and an unreadable `.wikitool-local.json` as an absent section. `doctor` reports each such "
"deviation as a `WARN` under `telemetry`, naming the field.",
)
REMOTES = Shape(
key=config.PUBLISH_REMOTES_FILENAME,
key="remotes",
title="Allowed push URLs",
purpose="Restricts the URLs `publish` may push to.",
fields=(
_SCHEMA,
Field(
"allowed_push_urls", "array of strings", "The push URLs `publish` may write to, "
"compared with the URL the remote resolves to for a push - not with its name. An "
"empty list allows no push at all.",
required=True, json_type="array", coerce=_str_list,
required=True, json_type="array", coerce=str_list, tier=CHECKOUT, gate=True,
),
),
examples=({"schema": 1, "allowed_push_urls": ["<url>"]},),
examples=({"allowed_push_urls": ["<url>"]},),
absent="Any remote is allowed; the Publish-Remote Gate does not apply.",
malformed="`publish` stops with an error before committing anything. A broken file is never "
malformed="`publish` stops with an error before committing anything. A broken section is never "
"read as \"no restriction\".",
unknown_text="Ignored.",
)
UPLOAD = Shape(
key=config.UPLOAD_CONFIG_FILENAME,
key="upload",
title="MCP submissions",
purpose="Turns on the MCP read server's `submit` tool and sets its limits.",
fields=(
_SCHEMA,
Field(
"identity_header", "string", "The request header the authenticating middleware sets to "
"the submitter's identity on every request.",
default="X-Forwarded-User", json_type="string", coerce=str, empty_is_missing=True,
lenient="An empty value means the default.",
lenient="An empty value means the default.", tier=CHECKOUT, gate=True,
),
_positive_integer("max_bytes", "Largest single submission, in bytes.", required=True),
positive_integer("max_bytes", "Largest single submission, in bytes.", required=True,
tier=CHECKOUT, gate=True),
Field(
"allowed_extensions", "array of strings", "The file extensions `submit` accepts, each "
"starting with `.`. Compared case-insensitively.",
required=True, json_type="array", coerce=_extensions, check=_dotted_extensions,
required=True, json_type="array", coerce=extensions, check=dotted_extensions,
lenient="Entries are read as strings and lower-cased; duplicates collapse.",
tier=CHECKOUT, gate=True,
),
Field(
"quota", "object", "Per-submitter limits over any rolling 24 hours.",
required=True, json_type="object",
fields=(
_positive_integer("submissions_per_day", "Submissions one submitter may make.",
required=True),
_positive_integer("bytes_per_day", "Bytes one submitter may submit.", required=True),
),
),
positive_integer("quota_submissions_per_day", "Submissions one submitter may make over any "
"rolling 24 hours.", required=True, tier=CHECKOUT, gate=True),
positive_integer("quota_bytes_per_day", "Bytes one submitter may submit over any rolling "
"24 hours.", required=True, tier=CHECKOUT, gate=True),
),
examples=({
"schema": 1, "identity_header": "X-Forwarded-User", "max_bytes": 10485760,
"identity_header": "X-Forwarded-User", "max_bytes": 10485760,
"allowed_extensions": [".md", ".pdf"],
"quota": {"submissions_per_day": 20, "bytes_per_day": 52428800},
"quota_submissions_per_day": 20, "quota_bytes_per_day": 52428800,
},),
absent="The `submit` tool is not registered: the write path does not exist.",
malformed="An error. The MCP server does not start and `doctor` reports `FAIL`. A broken file "
malformed="An error. The MCP server does not start and `doctor` reports `FAIL`. A broken section "
"is never read as \"no limits\".",
unknown_text="Ignored.",
)
TASKS = Shape(
key="tasks",
title="Task tracker",
purpose="Picks the task tracker the weekly review reads and writes through and sets the "
"review's thresholds. The provider's own connection settings sit in a section of the same "
"name.",
fields=(
Field("provider", "string", "Which adapter the review goes through.",
required=True, json_type="string", coerce=str, choices=PROVIDERS, tier=INSTANCE),
positive_integer("stalled_waiting_days", "A waiting-for item older than this many days "
"counts as overdue.", required=True, tier=INSTANCE),
positive_integer("unpaged_project_weeks", "A tracker project with no `kb/` page is "
"reported once it is older than this many weeks.", required=True,
tier=INSTANCE),
positive_integer("someday_stale_months", "A someday/maybe item untouched for this many "
"months counts as stale.", required=True, tier=INSTANCE),
),
examples=({
"provider": "superproductivity", "stalled_waiting_days": 14, "unpaged_project_weeks": 3,
"someday_stale_months": 5,
},),
absent="No tracker is configured - a valid state. The weekly review has nothing to check "
"against.",
malformed="An error, and `doctor` reports `FAIL`. A broken section is never read as \"no tracker "
"configured\".",
)
SUPERPRODUCTIVITY = Shape(
key="superproductivity",
title="Super Productivity",
purpose="Connection settings for the Super Productivity adapter.",
purpose="Connection settings for the Super Productivity adapter. `access` decides which of "
"the other fields apply: `api_base_url` and `api_token` with `api`, `backups_dir` with "
"`snapshot`.",
fields=(
Field(
"access", "string", "Which way in: `api`, the local REST API (reads the current state "
"and carries the write path), or `snapshot`, the newest backup snapshot (read-only).",
required=True, json_type="string",
required=True, json_type="string", choices=SUPERPRODUCTIVITY_ACCESS, tier=INSTANCE,
required_why="there is no default and no fallback between the two",
),
),
discriminator="access",
variants=(
("api", (
Field(
"api_base_url", "string", "Where the app's REST API listens.",
default="http://127.0.0.1:3876", json_type="string", coerce=str,
empty_is_missing=True, lenient="An empty value means the default.",
),
Field(
"api_token", "string", "The token from the app's settings.",
required=True, json_type="string", coerce=_exact_str, empty_is_missing=True,
required_why="every endpoint but GET /health requires Authorization: Bearer <token>",
),
)),
("snapshot", (
Field(
"backups_dir", "string", "The directory the app writes its backup snapshots to; "
"`~` is expanded.",
required=True, json_type="string", coerce=_exact_str, empty_is_missing=True,
required_why="the read path has nothing to read otherwise",
),
)),
Field(
"api_base_url", "string", "With `access: \"api\"`: where the app's REST API listens.",
default="http://127.0.0.1:3876", json_type="string", coerce=str,
empty_is_missing=True, lenient="An empty value means the default.",
tier=CHECKOUT, private=True,
),
Field(
"api_token", "string", "With `access: \"api\"`, required: the token from the app's "
"settings. Only ever a reference.", json_type="string", coerce=exact_str,
empty_is_missing=True, tier=SECRET,
required_why="every endpoint but GET /health requires Authorization: Bearer <token>",
),
Field(
"backups_dir", "string", "With `access: \"snapshot\"`, required: the directory the app "
"writes its backup snapshots to; `~` is expanded.",
json_type="string", coerce=exact_str, empty_is_missing=True, tier=CHECKOUT,
required_why="the read path has nothing to read otherwise",
),
),
examples=(
{"access": "api", "api_base_url": "http://127.0.0.1:3876", "api_token": "..."},
{"access": "api", "api_base_url": "http://127.0.0.1:3876", "api_token": {"ref": "sp_token"}},
{"access": "snapshot", "backups_dir": "~/.config/superProductivity/backups"},
),
unknown_keys=REJECT,
unknown_why="a section names only one access path's own fields",
unknown_text="Rejected: a section names only one access path's own fields, so a `backups_dir` "
"beside `access: \"api\"` is an error, not ignored.",
absent="Read only when `tasks.provider` is `superproductivity`.",
malformed="An error when it is read: a field of the other access path beside `access`, or a "
"missing `api_token` or `backups_dir`, stops the command with exit 1.",
)
CALDAV = Shape(
@@ -390,74 +372,34 @@ CALDAV = Shape(
title="CalDAV",
purpose="Connection settings for the CalDAV adapter.",
fields=(
Field("url", "string", "The account's CalDAV calendar home set. A trailing `/` is added "
"when missing.", required=True, json_type="string", coerce=_exact_str,
empty_is_missing=True),
Field("username", "string", "The login name, which can differ from the user id in the URL.",
required=True, json_type="string", coerce=_exact_str, empty_is_missing=True),
Field("app_password", "string", "An app password is recommended over the account password.",
required=True, json_type="string", coerce=_exact_str, empty_is_missing=True),
Field("inbox_list", "string", "The name of the one inbox list.",
required=True, json_type="string", coerce=_exact_str, empty_is_missing=True),
Field("someday_list", "string", "The name of the one someday/maybe list.",
required=True, json_type="string", coerce=_exact_str, empty_is_missing=True),
_required_string("url", "The account's CalDAV calendar home set. A trailing `/` is added "
"when missing.", tier=CHECKOUT, private=True),
_required_string("username", "The login name, which can differ from the user id in the "
"URL.", tier=CHECKOUT, private=True),
Field("app_password", "string", "An app password is recommended over the account "
"password. Only ever a reference.", required=True, json_type="string", coerce=exact_str,
empty_is_missing=True, tier=SECRET),
_required_string("inbox_list", "The name of the one inbox list.", tier=INSTANCE),
_required_string("someday_list", "The name of the one someday/maybe list.", tier=INSTANCE),
Field("exclude_lists", "array of strings", "Existing plain task lists that are not "
"projects.", default=[], json_type="array", coerce=_str_list),
"projects.", default=[], json_type="array", coerce=str_list, tier=INSTANCE),
),
examples=({
"url": "https://<host>/remote.php/dav/calendars/<user>/", "username": "<login>",
"app_password": "...", "inbox_list": "Inbox", "someday_list": "Someday",
"exclude_lists": ["..."],
"url": {"ref": "op://Chemenu/caldav/url"}, "username": "<login>",
"app_password": {"ref": "caldav_app_password"}, "inbox_list": "Inbox",
"someday_list": "Someday", "exclude_lists": ["..."],
},),
unknown_keys=REJECT,
unknown_text="Rejected.",
absent="Read only when `tasks.provider` is `caldav`.",
)
TASKS = Shape(
key=config.TASKS_CONFIG_FILENAME,
title="Task tracker",
purpose="Picks the task tracker the weekly review reads and writes through, its connection "
"settings and the review's thresholds.",
fields=(
_SCHEMA,
Field("provider", "string", "Which adapter the review goes through. The provider's own "
"settings sit in a section of the same name.",
required=True, json_type="string", coerce=str),
Field(
"thresholds", "object", "The weekly review's three thresholds.",
required=True, json_type="object",
fields=(
_positive_integer("stalled_waiting_days", "A waiting-for item older than this many "
"days counts as overdue.", required=True),
_positive_integer("unpaged_project_weeks", "A tracker project with no `kb/` page is "
"reported once it is older than this many weeks.", required=True),
_positive_integer("someday_stale_months", "A someday/maybe item untouched for this "
"many months counts as stale.", required=True),
),
),
),
section_key="provider",
sections=(SUPERPRODUCTIVITY, CALDAV),
examples=({
"schema": 1, "provider": "superproductivity",
"thresholds": {"stalled_waiting_days": 14, "unpaged_project_weeks": 3, "someday_stale_months": 5},
"superproductivity": {"access": "api", "api_base_url": "http://127.0.0.1:3876", "api_token": "..."},
},),
absent="No tracker is configured - a valid state. The weekly review has nothing to check "
"against.",
malformed="An error, and `doctor` reports `FAIL`. A broken file is never read as \"no tracker "
"configured\".",
unknown_text="Ignored, so the section of a provider not chosen right now can stay in the file.",
)
SHAPES = (TELEMETRY, REMOTES, UPLOAD, TASKS)
# The sections of the tiered settings files, `wikitool.json` and
# `.wikitool-local.json` (`chemenu.settings`). Each is a `Shape` whose `key` is
# the section's key in those files and whose every field declares its `tier`.
# Empty until the first feature reads its settings this way; `chemenu.settings`
# reads the tuple at call time, so a test can stand in a section of its own.
SETTINGS_SECTIONS: tuple[Shape, ...] = ()
# `.wikitool-local.json` (`chemenu.settings`), read at call time so a test can
# stand in a section of its own.
SETTINGS_SECTIONS: tuple[Shape, ...] = (TELEMETRY, REMOTES, UPLOAD, TASKS, SUPERPRODUCTIVITY, CALDAV)
# The sections `WIKITOOL_TASKS_CONFIG` replaces as a whole: a profile names a
# tracker, and everything that configures one.
PROFILE_SECTIONS = ("tasks", "superproductivity", "caldav")
# ---------------------------------------------------------------------------
@@ -484,104 +426,60 @@ def _article(noun: str) -> str:
return f"{'an' if noun[:1] in 'aeiou' else 'a'} {noun}"
def _check_fields(
shape: Shape, fields: tuple[Field, ...], data: dict, prefix: str, when: str, out: Checked,
values: dict,
) -> None:
for spec in fields:
path = prefix + spec.name
def check(shape: Shape, data: Any) -> Checked:
"""`data` read against `shape`: coerced values, problems, and what was
only accepted leniently. Never raises - each loader decides what a problem
means for it. A key no field names is listed in `unknown`, not a problem."""
out = Checked()
if not isinstance(data, dict):
out.problems.append(Problem("", f"must contain a JSON object, not {_described(data)}"))
return out
for spec in shape.fields:
raw = data.get(spec.name, _ABSENT)
if raw is _ABSENT or (spec.empty_is_missing and not raw):
if spec.required:
why = f" - {spec.required_why}" if spec.required_why else ""
empty = " or empty" if spec.empty_is_missing else ""
out.problems.append(Problem(path, f"{path} is missing{empty}, and required{when}{why}"))
out.problems.append(Problem(spec.name, f"{spec.name} is missing{empty}, and required{why}"))
else:
values[spec.name] = spec.default
continue
if spec.fields:
if not isinstance(raw, dict):
out.problems.append(Problem(path, f"{path} must be an object, got {_described(raw)}"))
continue
nested: dict = {}
before = len(out.problems)
_check_fields(shape, spec.fields, raw, f"{path}.", when, out, nested)
if len(out.problems) == before:
values[spec.name] = nested
out.given.add(path)
out.values[spec.name] = spec.default
continue
try:
value = spec.coerce(raw) if spec.coerce is not None else raw
except (TypeError, ValueError):
out.problems.append(Problem(path, f"{path} must be {_article(spec.type)}, got {_described(raw)}"))
out.problems.append(Problem(
spec.name, f"{spec.name} must be {_article(spec.type)}, got {_described(raw)}"))
continue
choices = shape.choices(spec)
if choices and value not in choices:
if spec.choices and value not in spec.choices:
why = f" - {spec.required_why}" if spec.required_why else ""
out.problems.append(Problem(
path, f"{path} must be one of {', '.join(repr(c) for c in choices)}{why}; got {raw!r}"
spec.name,
f"{spec.name} must be one of {', '.join(repr(c) for c in spec.choices)}{why}; got {raw!r}",
))
continue
problem = spec.check(value) if spec.check is not None else None
if problem:
out.problems.append(Problem(path, f"{path} {problem}"))
out.problems.append(Problem(spec.name, f"{spec.name} {problem}"))
continue
if spec.json_type is not None and not _JSON_TYPES[spec.json_type](raw):
out.coerced.append(Problem(
path, f"{path} is {_described(raw)}, not a JSON {spec.json_type}, and is read as "
f"{value!r}",
spec.name, f"{spec.name} is {_described(raw)}, not a JSON {spec.json_type}, and is "
f"read as {value!r}",
))
values[spec.name] = value
out.given.add(path)
def check(shape: Shape, data: Any) -> Checked:
"""`data` read against `shape`: coerced values, problems, and what was
only accepted leniently. Never raises - each loader decides what a problem
means for it."""
out = Checked()
if not isinstance(data, dict):
out.problems.append(Problem("", f"must contain a JSON object, not {_described(data)}"))
return out
_check_fields(shape, shape.fields, data, "", "", out, out.values)
known = {f.name for f in shape.fields}
if shape.discriminator:
chosen = out.values.get(shape.discriminator)
variant = shape.variant_fields(chosen) if shape.discriminator in out.given else None
if variant is None:
# The discriminator is already a problem; an "unknown key" on top
# would only name fields of a variant nobody picked.
return out
_check_fields(shape, variant, data, "", f" when {shape.discriminator}: {chosen!r}", out,
out.values)
known |= {f.name for f in variant}
if shape.section_key:
known |= {s.key for s in shape.sections}
chosen = out.values.get(shape.section_key)
if shape.section_key in out.given and not isinstance(data.get(chosen), dict):
out.problems.append(Problem(
chosen, f"missing or non-object {chosen!r} section holding that provider's own "
"connection settings",
))
out.unknown = sorted(set(data) - known)
if out.unknown and shape.unknown_keys == REJECT:
why = f" - {shape.unknown_why}" if shape.unknown_why else ""
out.problems.append(Problem("", f"does not take {out.unknown}{why}"))
out.values[spec.name] = value
out.given.add(spec.name)
out.unknown = sorted(set(data) - {spec.name for spec in shape.fields})
return out
def deviations(shape: Shape, data: Any) -> list[str]:
"""Everything in `data` that a strict reading of `shape` would not take as
written: problems, values accepted only by coercion, and unknown keys. For
a file whose loader is silent about all three - telemetry - this is what
a section whose loader is silent about all three - telemetry - this is what
`doctor` reports."""
checked = check(shape, data)
found = [p.message for p in checked.problems] + [p.message for p in checked.coerced]
if shape.unknown_keys == IGNORE:
found += [f"unknown key {key!r} is ignored" for key in checked.unknown]
found += [f"unknown key {key!r} is ignored" for key in checked.unknown]
return found
@@ -590,18 +488,12 @@ def deviations(shape: Shape, data: Any) -> list[str]:
# ---------------------------------------------------------------------------
def render_expected(shape: Shape) -> str:
"""The `Expected:` text of a loader's error message: the record's example,
or its examples joined by "or" where a discriminator offers several."""
return " or ".join(json.dumps(example) for example in shape.examples)
def _cell(text: str) -> str:
return text.replace("|", "\\|").replace("\n", " ")
def _default_cell(spec: Field) -> str:
if spec.required or spec.fields:
if spec.required:
return "-"
if spec.default_text:
return spec.default_text
@@ -610,53 +502,34 @@ def _default_cell(spec: Field) -> str:
return f"`{json.dumps(spec.default)}`"
def _rows(fields: tuple[Field, ...], prefix: str = "") -> list[str]:
rows = []
for spec in fields:
path = prefix + spec.name
def _tier_cell(spec: Field) -> str:
flags = [name for name, on in (("private", spec.private), ("names a command", spec.names_command),
("gate", spec.gate)) if on]
return ", ".join([spec.tier, *flags])
def render_markdown(shape: Shape) -> str:
"""The reference text for one section: its fields with their tiers, what
an absent and a malformed section mean, and an example."""
lines = [
"| Field | Tier | Type | Required | Default | Description |",
"|-------|------|------|----------|---------|-------------|",
]
for spec in shape.fields:
required = "yes" if spec.required else "no"
if spec.required and spec.empty_is_missing:
required = "yes, not empty"
description = " ".join(part for part in (spec.description, spec.lenient) if part)
rows.append(
f"| `{path}` | {spec.type} | {required} | {_cell(_default_cell(spec))} | {_cell(description)} |"
lines.append(
f"| `{spec.name}` | {_tier_cell(spec)} | {spec.type} | {required} | "
f"{_cell(_default_cell(spec))} | {_cell(description)} |"
)
rows += _rows(spec.fields, f"{path}.")
return rows
def _table(fields: tuple[Field, ...]) -> list[str]:
return ["| Field | Type | Required | Default | Description |",
"|-------|------|----------|---------|-------------|", *_rows(fields)]
def _example(example: dict) -> list[str]:
return ["```json", json.dumps(example, indent=2), "```"]
def _render_section(shape: Shape) -> list[str]:
lines = [f"**The `{shape.key}` section.** {shape.purpose}", "", *_table(shape.fields), ""]
for value, fields in shape.variants:
lines += [f"With `{shape.discriminator}: \"{value}\"`:", "", *_table(fields), ""]
lines += [f"- **Unknown keys:** {shape.unknown_text}", ""]
lines.append("")
if shape.absent:
lines.append(f"- **Absent:** {shape.absent}")
if shape.malformed:
lines.append(f"- **Malformed:** {shape.malformed}")
lines += ["", "Example:" if len(shape.examples) == 1 else "Examples:", ""]
for example in shape.examples:
lines += [*_example({shape.key: example}), ""]
return lines
def render_markdown(shape: Shape) -> str:
"""The reference text for one file: its fields, what an absent and a
malformed file mean, what happens to unknown keys, and an example."""
lines = [*_table(shape.fields), ""]
lines += [
f"- **Absent:** {shape.absent}",
f"- **Malformed:** {shape.malformed}",
f"- **Unknown keys:** {shape.unknown_text}",
"",
]
for section in shape.sections:
lines += _render_section(section)
lines += ["A complete example:" if shape.sections else "Example:", ""]
for example in shape.examples:
lines += [*_example(example), ""]
lines += ["```json", json.dumps({shape.key: example}, indent=2), "```", ""]
return "\n".join(lines).rstrip("\n") + "\n"
tools/chemenu/legacy_settings.py
+439
View File
@@ -0,0 +1,439 @@
"""The four per-feature settings files of earlier versions, and the move out of them.
Before the tiered settings (`chemenu.settings`) every feature had a file of its
own at the root of the checkout. None of them is read any more; this is the one
module that still names them, to detect a leftover and to carry its content
into `wikitool.json`, `.wikitool-local.json` and the secrets file -
`wikitool config migrate`.
| Old file | Becomes |
|---|---|
| `.wikitool-telemetry.json` | section `telemetry` |
| `.wikitool-remotes.json` | section `remotes` |
| `.wikitool-upload.json` | section `upload` (`quota.*` is flat: `quota_*`) |
| `.wikitool-tasks.json` | sections `tasks` (`thresholds.*` is flat), `superproductivity`, `caldav` |
What the migration promises, and how it keeps it:
- **Every setting keeps its effective value, defaults included.** The old
content is read with the same coercions and checks the new sections use
(`config_shapes.check`), so a value is accepted exactly when it was, and what
is written is the coerced value. A field the old file did not set is not
written: its default still applies.
- **No secret value reaches the tree.** A secret field's value goes into the
secrets file (`config.secrets_file`), only a reference into `wikitool.json`;
nothing here prints a value.
- **An abort leaves the old files as they were.** Everything is computed and
checked in memory first. Then the targets are written - secrets file,
`wikitool.json`, `.wikitool-local.json` - and read back and checked again.
Only then are the old files removed; before that, any failure puts every
target back to the bytes it had.
"""
from __future__ import annotations
import copy
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Optional
from chemenu import config, config_shapes, settings
from chemenu.config_shapes import INSTANCE, SECRET
from chemenu.errors import ChemenuError, ValidationError
TELEMETRY_FILE = ".wikitool-telemetry.json"
REMOTES_FILE = ".wikitool-remotes.json"
UPLOAD_FILE = ".wikitool-upload.json"
TASKS_FILE = ".wikitool-tasks.json"
# Feature (= the section that replaces the file) -> the file it came from.
FILES = {
"telemetry": TELEMETRY_FILE,
"remotes": REMOTES_FILE,
"upload": UPLOAD_FILE,
"tasks": TASKS_FILE,
}
MIGRATE_COMMAND = "tools/wikitool config migrate"
class LegacySettingsLeftover(ChemenuError):
"""A settings file of an earlier version is still there and nothing reads
it. Not a fault in the caller's input and not something to retry: a human
runs `config migrate`, so the CLI renders it as exit 42 (`needs_clearance`)."""
def __init__(self, names: list[str]):
self.names = names
listed = ", ".join(names)
verb = "is a settings file" if len(names) == 1 else "are settings files"
super().__init__(
f"{listed} {verb} of an earlier version and no longer read - the settings it held "
f"are not in effect. Move them into the tiered settings with: {MIGRATE_COMMAND}"
)
def present(root: "Path | str") -> list[str]:
"""The old files that exist under `root`, in a fixed order."""
return [name for name in FILES.values() if (Path(root) / name).exists()]
def refuse_leftover(root: "Path | str", *features: str) -> None:
"""Raise `LegacySettingsLeftover` when the old file of any of `features`
exists. The tracker file is not looked at while a tracker profile
(`WIKITOOL_TASKS_CONFIG`) replaces its sections - it is not what is read."""
names = []
for feature in features:
if feature == "tasks" and config.tasks_config_override() is not None:
continue
name = FILES[feature]
if (Path(root) / name).exists():
names.append(name)
if names:
raise LegacySettingsLeftover(names)
# ---------------------------------------------------------------------------
# Reading the old files
# ---------------------------------------------------------------------------
@dataclass
class Plan:
"""What the migration would do: the values per section (coerced, JSON-safe),
and what to say about them."""
entries: dict[str, dict[str, Any]] = field(default_factory=dict)
notes: list[str] = field(default_factory=list)
files: list[Path] = field(default_factory=list)
def _json_safe(value: Any) -> Any:
return list(value) if isinstance(value, tuple) else value
def _read_old(path: Path, *, lenient: bool, plan: Plan) -> Any:
"""The old file's parsed JSON. A telemetry file that cannot be read counts
as absent, as it always did; any other raises `ValidationError`."""
import json
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError) as exc:
if lenient:
plan.notes.append(f"{path.name} could not be read ({type(exc).__name__}) and counts as absent, as before")
return None
raise ValidationError(f"{path.name} cannot be read as JSON ({exc}) - fix or remove it, then run again") from None
def _take(plan: Plan, shape: config_shapes.Shape, data: Any, *, file: str, lenient: bool = False) -> None:
"""The coerced values of `data` that `shape` accepts, into `plan`. Strict:
a problem raises `ValidationError` naming the old file. Lenient: a field
with a problem is dropped and said so."""
checked = config_shapes.check(shape, data)
if checked.problems:
found = "; ".join(problem.message for problem in checked.problems)
if not lenient:
raise ValidationError(f"{file}: {found}")
plan.notes.append(f"{file}: not migrated, as it was not read - {found}")
values = {name: _json_safe(checked.values[name]) for name in checked.given}
ignored = [key for key in checked.unknown if key != "schema"]
if ignored:
plan.notes.append(f"{file}: {', '.join(ignored)} was not read before and is not migrated")
if values:
plan.entries.setdefault(shape.key, {}).update(values)
def _flat(data: Any, nested: str, prefix: str, file: str) -> Any:
"""`data` with the object `nested` spread into `<prefix>_<key>` keys."""
if not isinstance(data, dict) or nested not in data:
return data
inner = data[nested]
if not isinstance(inner, dict):
raise ValidationError(f"{file}: {nested} must be an object")
flat = {key: value for key, value in data.items() if key != nested}
for key, value in inner.items():
flat[f"{prefix}{key}"] = value
return flat
def _provider_section(name: str, body: Any, file: str) -> dict[str, Any]:
"""The migrated values of one provider section, after the checks the
adapter makes when it reads it - the access-path rules of Super
Productivity included. Raises `ValidationError`."""
shape = {config_shapes.SUPERPRODUCTIVITY.key: config_shapes.SUPERPRODUCTIVITY,
config_shapes.CALDAV.key: config_shapes.CALDAV}[name]
if not isinstance(body, dict):
raise ValidationError(f"{file}: {name} must be an object")
try:
section = settings.Section.from_data(shape, body)
if name == config_shapes.SUPERPRODUCTIVITY.key:
from chemenu.tasks import superproductivity
superproductivity.SuperProductivityConfig.from_section(section)
except ValidationError as exc:
raise ValidationError(f"{file}: {name}: {exc}") from None
return {key: _json_safe(section.values[key]) for key in section.given}
def read_plan(root: "Path | str") -> Plan:
"""What `config migrate` would move, from every old file under `root`.
Raises `ValidationError` for the first old file that is broken; nothing is
written by then."""
root = Path(root)
plan = Plan()
path = root / TELEMETRY_FILE
if path.exists():
plan.files.append(path)
data = _read_old(path, lenient=True, plan=plan)
if data is not None and not isinstance(data, dict):
plan.notes.append(f"{TELEMETRY_FILE} is not a JSON object and counts as absent, as before")
elif data is not None:
_take(plan, config_shapes.TELEMETRY, data, file=TELEMETRY_FILE, lenient=True)
path = root / REMOTES_FILE
if path.exists():
plan.files.append(path)
_take(plan, config_shapes.REMOTES, _read_old(path, lenient=False, plan=plan), file=REMOTES_FILE)
path = root / UPLOAD_FILE
if path.exists():
plan.files.append(path)
data = _flat(_read_old(path, lenient=False, plan=plan), "quota", "quota_", UPLOAD_FILE)
_take(plan, config_shapes.UPLOAD, data, file=UPLOAD_FILE)
path = root / TASKS_FILE
if path.exists():
plan.files.append(path)
_read_tasks(path, plan)
return plan
def _read_tasks(path: Path, plan: Plan) -> None:
data = _read_old(path, lenient=False, plan=plan)
data = _flat(data, "thresholds", "", TASKS_FILE)
# `thresholds` was an object whose keys are now the section's own fields;
# `provider` was the only other field. Everything else is a provider section.
_take(plan, config_shapes.TASKS, {k: v for k, v in data.items() if k not in config_shapes.PROVIDERS}
if isinstance(data, dict) else data, file=TASKS_FILE)
provider = plan.entries["tasks"]["provider"]
for name in config_shapes.PROVIDERS:
body = data.get(name)
if name == provider:
if body is None:
raise ValidationError(f"{TASKS_FILE}: provider is {provider!r}, so the file needs a {provider} section")
plan.entries[name] = _provider_section(name, body, TASKS_FILE)
elif body is not None:
try:
plan.entries[name] = _provider_section(name, body, TASKS_FILE)
except ValidationError as exc:
plan.notes.append(f"{exc} - the section of a provider that is not chosen is not migrated")
# ---------------------------------------------------------------------------
# Splitting by tier, merging into what is there
# ---------------------------------------------------------------------------
@dataclass
class Targets:
instance: dict
local: dict
secrets: dict
moved: list[str]
def _secret_name(section: str, name: str) -> str:
return f"{section}_{name}"
def _merge(target: dict, section: Optional[str], name: str, value: Any, label: str) -> None:
body = target
where = name
if section is not None:
body = target.setdefault(section, {})
where = f"{section}.{name}"
if not isinstance(body, dict):
raise ValidationError(f"{label}: {section} is not an object")
if name in body and body[name] != value:
raise ValidationError(
f"{label}: {where} already holds a different value than the old file. Decide which "
"one stands, remove the other, and run again"
)
body[name] = value
def split(plan: Plan, instance: dict, local: dict, secrets: dict) -> Targets:
"""`plan` distributed over the files by tier, on top of what they hold.
Raises `ValidationError` on a key that already holds another value."""
instance, local, secrets = copy.deepcopy(instance), copy.deepcopy(local), dict(secrets)
known = settings.sections()
moved: list[str] = []
for section, values in plan.entries.items():
shape = known[section]
for name, value in values.items():
spec = shape.field(name)
key = f"{section}.{name}"
if spec.tier == SECRET:
ref = _secret_name(section, name)
if secrets.get(ref, value) != value:
raise ValidationError(
f"the secrets file already holds a different value under {ref!r} - decide "
"which one stands, remove the other, and run again"
)
secrets[ref] = value
_merge(instance, section, name, {settings.REF_KEY: ref}, config.INSTANCE_SETTINGS_FILENAME)
moved.append(f"{key} -> secrets file, reference {ref!r} in {config.INSTANCE_SETTINGS_FILENAME}")
elif spec.tier == INSTANCE:
_merge(instance, section, name, value, config.INSTANCE_SETTINGS_FILENAME)
moved.append(f"{key} -> {config.INSTANCE_SETTINGS_FILENAME}")
else:
_merge(local, section, name, value, config.LOCAL_SETTINGS_FILENAME)
moved.append(f"{key} -> {config.LOCAL_SETTINGS_FILENAME}")
return Targets(instance, local, secrets, moved)
# ---------------------------------------------------------------------------
# Writing, reading back, rolling back
# ---------------------------------------------------------------------------
def _write(path: Path, data: dict) -> None:
settings.write_file(path, data)
def _missing_ancestors(path: Path) -> list[Path]:
found = []
parent = path.parent
while not parent.exists():
found.append(parent)
parent = parent.parent
return found
def _restore(originals: dict[Path, Optional[bytes]], created_dirs: list[Path]) -> None:
for path, content in originals.items():
try:
if content is None:
path.unlink(missing_ok=True)
else:
path.write_bytes(content)
except OSError:
pass
for directory in created_dirs:
try:
directory.rmdir()
except OSError:
pass
def _normal(value: Any) -> Any:
return list(value) if isinstance(value, tuple) else value
def _verify(root: Path, plan: Plan, secrets_path: Optional[Path], expected_secrets: dict) -> None:
"""Read the written files back and check that every planned value is in
effect. Raises `ValidationError`."""
loaded = settings.check_files(settings.read_file(settings.instance_path(root)),
settings.read_file(settings.local_path(root)))
known = settings.sections()
for section, values in plan.entries.items():
got = loaded.section(section)
for name, value in values.items():
spec = known[section].field(name)
if spec.tier == SECRET:
ref = got.refs.get(name)
if ref is None or ref.target != _secret_name(section, name):
raise ValidationError(f"read-back: {section}.{name} is not a reference to the secrets file")
elif _normal(got.values.get(name)) != value:
raise ValidationError(f"read-back: {section}.{name} does not hold the migrated value")
if expected_secrets:
assert secrets_path is not None
stored = settings.read_secrets_file(secrets_path) or {}
for ref, value in expected_secrets.items():
if stored.get(ref) != value:
raise ValidationError(f"read-back: the secrets file does not hold {ref!r}")
@dataclass
class Result:
moved: list[str]
notes: list[str]
removed: list[str]
def migrate(root: "Path | str") -> Result:
"""Carry every old file under `root` into the tiered settings. Raises
`ValidationError` - with the old files and the targets as they were - for
anything it cannot do or check; returns an empty `Result` when there is
nothing to move."""
root = Path(root)
plan = read_plan(root)
if not plan.files:
return Result([], [], [])
instance_path, local_path = settings.instance_path(root), settings.local_path(root)
instance = settings.read_file(instance_path)
local = settings.read_file(local_path)
# Everything that can be known without writing is checked first.
instance_id = (instance or {}).get(settings.INSTANCE_ID_KEY)
needs_secrets = any(_tier(section, name) == SECRET
for section, values in plan.entries.items() for name in values)
secrets_path: Optional[Path] = None
existing_secrets: dict = {}
if needs_secrets:
if instance_id is None:
instance_id = settings.new_instance_id()
secrets_path = config.secrets_file(instance_id)
existing_secrets = settings.read_secrets_file(secrets_path) or {}
targets = split(plan, instance or {}, local or {}, existing_secrets)
changed_secrets = {ref: value for ref, value in targets.secrets.items()
if existing_secrets.get(ref) != value}
write_instance = targets.instance != (instance or {})
write_local = targets.local != (local or {})
if write_instance:
targets.instance.setdefault("schema", settings.SCHEMA)
targets.instance.setdefault(settings.INSTANCE_ID_KEY, instance_id or settings.new_instance_id())
if write_local:
targets.local.setdefault("schema", settings.SCHEMA)
try:
settings.check_files(targets.instance if (write_instance or instance is not None) else None,
targets.local if (write_local or local is not None) else None)
except ValidationError as exc:
raise ValidationError(f"the settings would not pass their check after the move:\n{exc}") from None
writes: list[tuple[Path, dict]] = []
if changed_secrets:
assert secrets_path is not None
writes.append((secrets_path, targets.secrets))
if write_instance:
writes.append((instance_path, targets.instance))
if write_local:
writes.append((local_path, targets.local))
originals = {path: (path.read_bytes() if path.exists() else None) for path, _ in writes}
created_dirs = _missing_ancestors(secrets_path) if changed_secrets and secrets_path else []
try:
for path, data in writes:
_write(path, data)
_verify(root, plan, secrets_path, {ref: targets.secrets[ref] for ref in changed_secrets})
except BaseException as exc:
_restore(originals, created_dirs)
if isinstance(exc, ValidationError):
raise ValidationError(f"{exc}. Nothing was changed; the old files are as they were") from None
if isinstance(exc, Exception):
raise ValidationError(
f"the move failed ({type(exc).__name__}). Nothing was changed; the old files are as they were"
) from None
raise
removed = []
for path in plan.files:
path.unlink()
removed.append(path.name)
return Result(targets.moved, plan.notes, removed)
def _tier(section: str, name: str) -> str:
return settings.sections()[section].field(name).tier
tools/chemenu/mcp/__main__.py
+6
View File
@@ -20,6 +20,7 @@ from __future__ import annotations
import argparse
import sys
from chemenu.legacy_settings import LegacySettingsLeftover
from chemenu.mcp.server import TRANSPORTS, TraceWouldWriteIntoCorpus, serve
@@ -72,6 +73,11 @@ def main(argv: list[str] | None = None) -> int:
# be a surprise buried in a log.
print(f"ERROR {exc}", file=sys.stderr)
return 1
except LegacySettingsLeftover as exc:
# An old upload file would otherwise read as "no upload configured" -
# and the server would start without the `submit` tool its operator set up.
print(f"ERROR {exc}", file=sys.stderr)
return 1
return 0
tools/chemenu/mcp/server.py
+5 -5
View File
@@ -15,8 +15,8 @@ test asserts it by importing this module in a clean interpreter and looking at
`sys.modules`.
**The sixth, `submit`, writes by a positive list instead of an absence.** It
exists only when `.wikitool-upload.json` opts this checkout into it
(`config.UPLOAD_CONFIG_FILENAME`, absent by default) - not merely hidden, but
exists only when the `upload` section of `.wikitool-local.json` opts this
checkout into it (absent by default) - not merely hidden, but
never registered on the server, the same distinction #19 draws for every
other write function. When it is registered, every byte it writes still goes
through exactly one choke point, `chemenu.upload._write_atomic_within`, which
@@ -128,10 +128,10 @@ def build_server(
if check_trace:
check_trace_destination(corpus.root)
# A malformed `.wikitool-upload.json` is a start failure, not "no limits":
# this file decides whether an unauthenticated write path is offered at
# A malformed `upload` section is a start failure, not "no limits":
# this section decides whether an unauthenticated write path is offered at
# all, so it is read once, up front, rather than lazily inside the tool
# closure where a broken file would only surface on the first `submit`
# closure where a broken section would only surface on the first `submit`
# call. `upload_cfg` is None exactly when the tool below is not defined.
upload_cfg = upload.read_config(corpus.root)
tools/chemenu/reference_doc.py
+2
View File
@@ -65,6 +65,8 @@ CONFIG_REGIONS = (
("reference-config-telemetry", "TELEMETRY"),
("reference-config-remotes", "REMOTES"),
("reference-config-tasks", "TASKS"),
("reference-config-superproductivity", "SUPERPRODUCTIVITY"),
("reference-config-caldav", "CALDAV"),
("reference-config-upload", "UPLOAD"),
)
tools/chemenu/review.py
+2 -2
View File
@@ -134,7 +134,7 @@ def run_review(root: Path, *, today: Optional[date] = None) -> ReviewReport:
provider read is a fresh call, and no file under `root` is touched.
Raises `ValidationError` for anything that is not "the provider is
unreachable right now" - no `.wikitool-tasks.json` at all, or a
unreachable right now" - no `tasks` section at all, or a
malformed one. Those are configuration problems a retry cannot fix, so
`commands.review_cmd` reports them as an ordinary exit-1 error rather than
a partial report.
@@ -143,7 +143,7 @@ def run_review(root: Path, *, today: Optional[date] = None) -> ReviewReport:
cfg = tasks_config.read_config(root)
if cfg is None:
raise ValidationError(
f"No {config.TASKS_CONFIG_FILENAME} - no task tracker is configured, so there is "
"No tasks section in the settings - no task tracker is configured, so there is "
"nothing to join the weekly review against. Configure one first."
)
reader = build_reader(cfg)
tools/chemenu/settings.py
+125 -18
View File
@@ -27,6 +27,12 @@ in the local file only: an instance that is public commits `wikitool.json`
publicly. As a reference it may sit in either, which is what makes the committed
file portable across every machine with the same 1Password account.
**A profile** (`WIKITOOL_TASKS_CONFIG`) replaces the `tasks`, `superproductivity`
and `caldav` sections of both files as a whole: it is a file in the format of
`.wikitool-local.json` that holds only those sections. It is checked like the
local file, except that a plain `instance`-tier value may stand in it - a
profile is one file, not a tier - and a secret is a reference there too.
The sections themselves are `config_shapes.SETTINGS_SECTIONS`, read at call
time so a test can stand in a section of its own. Unknown top-level keys are
reported (`unknown`), not refused: a section a newer stack writes must not stop
@@ -58,6 +64,7 @@ from chemenu.errors import ChemenuError, ValidationError
SCHEMA = 1
INSTANCE_ID_KEY = "instance_id"
REF_KEY = "ref"
PROFILE_LIVE_KEY = "live_test_version"
OP_SCHEME = "op://"
OP_TIMEOUT_SECONDS = 60
@@ -140,13 +147,10 @@ def parse_ref(value: Any) -> Ref:
def section_problems(shape: Shape) -> list[str]:
"""What keeps `shape` from being a settings section: every field tiered,
no discriminator and no sub-sections - a field's tier has to be readable off
the field alone."""
"""What keeps `shape` from being a settings section: every field tiered, so
a field's tier can be read off the field alone."""
where = f"settings section {shape.key!r}"
problems = [f"{where}: field {spec.name!r} has no tier" for spec in shape.fields if not spec.tier]
if shape.discriminator or shape.sections:
problems.append(f"{where}: a settings section takes neither variants nor sub-sections")
if shape.key in (INSTANCE_ID_KEY, "schema"):
problems.append(f"{where}: {shape.key!r} is a key of the file itself")
return problems
@@ -215,10 +219,33 @@ def read_file(path: Path) -> Optional[dict]:
return data
PROFILE_LABEL = "the tracker profile"
def _file_label(in_instance: bool) -> str:
return config.INSTANCE_SETTINGS_FILENAME if in_instance else config.LOCAL_SETTINGS_FILENAME
def profile_placement_problem(spec: Field, value: Any, key: str) -> Optional[str]:
"""Why `value` may not stand for `spec` in a profile, or None. A profile is
one file, so no tier says where a plain value goes - but a secret is still
only ever a reference."""
if is_ref(value):
problem = ref_problem(value)
if problem:
return f"{PROFILE_LABEL}: {key} {problem}"
if not spec.takes_ref:
return (f"{PROFILE_LABEL}: {key} takes a plain value, not a reference - only a secret "
"or a private field takes one")
return None
if spec.tier == SECRET:
return (f"{PROFILE_LABEL}: {key} is tier {SECRET!r} and holds a plain value - a secret is "
f'only ever a reference, {{"{REF_KEY}": "<name>"}} or {{"{REF_KEY}": "{OP_SCHEME}..."}}; '
f"the value itself goes into {config.ENV_SECRET_PREFIX}<NAME>, the secrets file "
"or 1Password")
return None
def placement_problem(spec: Field, value: Any, in_instance: bool, key: str) -> Optional[str]:
"""Why `value` may not stand for `spec` in this file, or None. Never
quotes the value: a plain secret is exactly what must not be printed."""
@@ -293,6 +320,30 @@ class Section:
return self[name]
return default
@classmethod
def from_data(cls, shape: Shape, data: dict, instance_id: Optional[str] = None) -> "Section":
"""A section from one flat object, plain values and references side by
side - for code and tests that hold a section's content without the
files around it. Raises `ValidationError` listing every problem."""
fields = {spec.name for spec in shape.fields}
problems = [f"{key} is not a field of {shape.key!r} (fields: {', '.join(sorted(fields))})"
for key in data if key not in fields]
for key, value in data.items():
if is_ref(value):
problem = ref_problem(value)
if problem:
problems.append(f"{key} {problem}")
if problems:
raise ValidationError("; ".join(problems))
plain = {k: v for k, v in data.items() if not is_ref(v)}
refs = {k: Ref(v[REF_KEY]) for k, v in data.items() if is_ref(v)}
checked = config_shapes.check(shape, plain)
found = [p.message for p in checked.problems if p.field not in refs]
if found:
raise ValidationError("; ".join(found))
return cls(shape, values=checked.values, refs=refs,
given=set(checked.given) | set(refs), instance_id=instance_id)
@dataclass
class Settings:
@@ -318,29 +369,59 @@ class Settings:
yield f"{name}.{field_name}", ref
def _profile_envelope_problems(data: dict) -> list[str]:
problems = []
if "schema" in data and data["schema"] != SCHEMA:
problems.append(f"{PROFILE_LABEL}: schema must be {SCHEMA}")
allowed = {"schema", PROFILE_LIVE_KEY, *config_shapes.PROFILE_SECTIONS}
for name in data:
if name not in allowed:
problems.append(
f"{PROFILE_LABEL}: {name} is not a key a profile holds (sections: "
f"{', '.join(config_shapes.PROFILE_SECTIONS)}; also schema and {PROFILE_LIVE_KEY})"
)
return problems
def check_files(
instance_data: Optional[dict], local_data: Optional[dict],
known: Optional[dict[str, Shape]] = None, *, complete: bool = True,
profile_data: Optional[dict] = None,
) -> Settings:
"""Both files' contents checked against the registered sections. Raises
`ValidationError` listing every problem, one per line; resolves nothing.
`complete=False` is the check for a write in progress (`config set`): a
required field nobody has set yet is not a problem there, or the first
value of a section could never be written."""
value of a section could never be written.
`profile_data` is a tracker profile: its sections take the place of the
profile sections of both files, which are then not read at all."""
known = sections() if known is None else known
problems = _envelope_problems(instance_data, True) + _envelope_problems(local_data, False)
if profile_data is not None:
problems += _profile_envelope_problems(profile_data)
settings = Settings(instance_present=instance_data is not None, local_present=local_data is not None)
if instance_data is not None and isinstance(instance_data.get(INSTANCE_ID_KEY), str):
settings.instance_id = instance_data[INSTANCE_ID_KEY]
merged: dict[str, dict[str, tuple[Any, bool]]] = {}
for data, in_instance in ((instance_data, True), (local_data, False)):
if data is None:
continue
here = _file_label(in_instance)
# section -> field -> (value, label of the file it stands in)
merged: dict[str, dict[str, tuple[Any, str]]] = {}
sources: list[tuple[dict, Optional[bool]]] = [
(data, in_instance) for data, in_instance in ((instance_data, True), (local_data, False))
if data is not None
]
if profile_data is not None:
sources.append((profile_data, None))
for data, in_instance in sources:
is_profile = in_instance is None
here = PROFILE_LABEL if is_profile else _file_label(in_instance)
for name, body in data.items():
if name in ("schema", INSTANCE_ID_KEY):
if name in ("schema", INSTANCE_ID_KEY, PROFILE_LIVE_KEY):
continue
if profile_data is not None and not is_profile and name in config_shapes.PROFILE_SECTIONS:
continue
if is_profile and name not in config_shapes.PROFILE_SECTIONS:
continue
shape = known.get(name)
if shape is None:
@@ -357,7 +438,8 @@ def check_files(
problems.append(f"{here}: {key} is not a field of {name!r} "
f"(fields: {', '.join(fields)})")
continue
problem = placement_problem(spec, value, in_instance, key)
problem = (profile_placement_problem(spec, value, key) if is_profile
else placement_problem(spec, value, in_instance, key))
if problem:
problems.append(problem)
continue
@@ -365,7 +447,7 @@ def check_files(
problems.append(f"{key} is set in both {config.INSTANCE_SETTINGS_FILENAME} and "
f"{config.LOCAL_SETTINGS_FILENAME} - a value has exactly one place")
continue
merged[name][field_name] = (value, in_instance)
merged[name][field_name] = (value, here)
for name, entries in merged.items():
shape = known[name]
@@ -378,7 +460,7 @@ def check_files(
if not complete and problem.field not in entries:
continue # not set yet - the write in progress may be the first of several
origin = entries.get(problem.field)
where = f"{_file_label(origin[1])}: " if origin else ""
where = f"{origin[1]}: " if origin else ""
problems.append(f"{where}{name}.{problem.message}")
settings.by_section[name] = Section(
shape, values=checked.values, refs=refs,
@@ -411,9 +493,34 @@ def committed_file_problems(instance_data: dict, known: Optional[dict[str, Shape
return problems
def load(root: "Path | str | None" = None) -> Settings:
"""Both files under `root` (default `config.ROOT`), read and checked."""
return check_files(read_file(instance_path(root)), read_file(local_path(root)))
def read_profile() -> Optional[dict]:
"""The tracker profile `$WIKITOOL_TASKS_CONFIG` names, or None when it is
unset. Raises `ValidationError` when it names no readable JSON object."""
path = config.tasks_config_override()
if path is None:
return None
if not path.is_file():
raise ValidationError(f"{config.ENV_TASKS_CONFIG} points at {path}, which is not a file")
try:
return read_file(path)
except ValidationError as exc:
raise ValidationError(f"{config.ENV_TASKS_CONFIG} -> {exc}") from None
def load(root: "Path | str | None" = None, only: Optional[tuple[str, ...]] = None) -> Settings:
"""Both files under `root` (default `config.ROOT`), read and checked - plus
the tracker profile, when `$WIKITOOL_TASKS_CONFIG` names one.
`only` limits the check to the named sections, so a command that needs one
feature is not stopped by a broken section of another."""
known = sections()
if only is not None:
known = {name: shape for name, shape in known.items() if name in only}
profile = None
if only is None or set(only) & set(config_shapes.PROFILE_SECTIONS):
profile = read_profile()
return check_files(read_file(instance_path(root)), read_file(local_path(root)), known,
profile_data=profile)
# ---------------------------------------------------------------------------
tools/chemenu/tasks/__init__.py
+4 -4
View File
@@ -31,14 +31,14 @@ def build_reader(cfg: TasksConfig) -> TaskReader:
if cfg.provider == "superproductivity":
from chemenu.tasks import superproductivity as sp
sp_cfg = sp.SuperProductivityConfig.from_dict(cfg.provider_config)
sp_cfg = sp.SuperProductivityConfig.from_section(cfg.provider_config)
if sp_cfg.access == sp.ACCESS_API:
return sp.SuperProductivityApiReader(sp_cfg)
return sp.SuperProductivityReader(sp_cfg)
if cfg.provider == "caldav":
from chemenu.tasks import caldav as cd
cd_cfg = cd.CalDAVConfig.from_dict(cfg.provider_config)
cd_cfg = cd.CalDAVConfig.from_section(cfg.provider_config)
return cd.CalDAVReader(cd_cfg)
raise ValidationError(f"No reader is wired up for task provider {cfg.provider!r}.")
@@ -58,7 +58,7 @@ def build_writer(cfg: TasksConfig, reader: TaskReader) -> TaskWriter:
if cfg.provider == "superproductivity":
from chemenu.tasks import superproductivity as sp
sp_cfg = sp.SuperProductivityConfig.from_dict(cfg.provider_config)
sp_cfg = sp.SuperProductivityConfig.from_section(cfg.provider_config)
if sp_cfg.access != sp.ACCESS_API:
raise ValidationError(
"superproductivity: the tracker is read-only from here (access: "
@@ -69,6 +69,6 @@ def build_writer(cfg: TasksConfig, reader: TaskReader) -> TaskWriter:
if cfg.provider == "caldav":
from chemenu.tasks import caldav as cd
cd_cfg = cd.CalDAVConfig.from_dict(cfg.provider_config)
cd_cfg = cd.CalDAVConfig.from_section(cfg.provider_config)
return cd.CalDAVWriter(cd_cfg, reader)
raise ValidationError(f"No writer is wired up for task provider {cfg.provider!r}.")
tools/chemenu/tasks/caldav.py
+10 -17
View File
@@ -82,7 +82,7 @@ import icalendar
import urllib.error
import urllib.request
from chemenu import config_shapes
from chemenu import settings
from chemenu.errors import ValidationError
from chemenu.tasks.protocol import (
OpenItem,
@@ -105,7 +105,7 @@ SOURCE_KIND = "api"
@dataclass(frozen=True)
class CalDAVConfig:
"""This provider's own section of `.wikitool-tasks.json`
"""This provider's `caldav` section of the tiered settings
(`TasksConfig.provider_config`). `url` is normalized to always end in
`/` so every later `urljoin` treats it as a directory, never a document -
a bare `urljoin` on a slash-less base silently drops its last path
@@ -119,22 +119,15 @@ class CalDAVConfig:
exclude_lists: tuple[str, ...]
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "CalDAVConfig":
shape = config_shapes.CALDAV
checked = config_shapes.check(shape, data)
if checked.problems:
raise ValidationError(
f"caldav config: {checked.message()}. Expected: {config_shapes.render_expected(shape)}"
)
values = checked.values
url = values["url"] if values["url"].endswith("/") else values["url"] + "/"
def from_section(cls, section: "settings.Section") -> "CalDAVConfig":
url = section["url"]
return cls(
url=url,
username=values["username"],
app_password=values["app_password"],
inbox_list=values["inbox_list"],
someday_list=values["someday_list"],
exclude_lists=tuple(values["exclude_lists"]),
url=url if url.endswith("/") else url + "/",
username=section["username"],
app_password=section["app_password"],
inbox_list=section["inbox_list"],
someday_list=section["someday_list"],
exclude_lists=tuple(section["exclude_lists"]),
)
tools/chemenu/tasks/config.py
+32 -42
View File
@@ -1,25 +1,26 @@
"""Reads `.wikitool-tasks.json` (Gitea #124) - `config.TASKS_CONFIG_FILENAME`, or the
file `$WIKITOOL_TASKS_CONFIG` names instead (Gitea #156, `config.tasks_config_path`).
"""Reads the tracker's settings (Gitea #124) - the `tasks`, `superproductivity` and
`caldav` sections of the tiered settings (`chemenu.settings`), or the profile
`$WIKITOOL_TASKS_CONFIG` names instead (Gitea #156).
Same posture as `chemenu.upload.read_config`: absent means "no tracker
configured for this instance", a legitimate state that `doctor` reports as OK,
never as a fault. Malformed is a `ValidationError`, never silently ignored -
this file decides which provider (and which credentials) the review talks to,
so a broken one must not be read as "nothing configured".
these sections decide which provider (and which credentials) the review talks
to, so a broken one must not be read as "nothing configured". A file of an
earlier version that is still there raises `LegacySettingsLeftover`.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from chemenu import config, config_shapes
from chemenu import config_shapes, legacy_settings, settings
from chemenu.errors import ValidationError
# The providers this package ships an adapter for: the sections of the file's
# record (`config_shapes.TASKS`). Checked at read time so a typo in `provider`
# fails here, rather than surfacing later as an unhelpful "unknown provider"
# from whatever code tried to dispatch on it.
KNOWN_PROVIDERS = tuple(section.key for section in config_shapes.TASKS.sections)
# The providers this package ships an adapter for. Checked at read time so a
# typo in `provider` fails here, rather than surfacing later as an unhelpful
# "unknown provider" from whatever code tried to dispatch on it.
KNOWN_PROVIDERS = config_shapes.PROVIDERS
@dataclass(frozen=True)
@@ -36,47 +37,36 @@ class Thresholds:
@dataclass(frozen=True)
class TasksConfig:
"""The opt-in, read from `.wikitool-tasks.json`."""
"""The opt-in, read from the tiered settings."""
provider: str
thresholds: Thresholds
provider_config: dict[str, Any]
provider_config: "settings.Section"
def read_config(root: "Any") -> "TasksConfig | None":
"""The tracker configuration for `root`, or `None` when the file is
absent - which means no tracker is configured for this instance, not that
one failed to load."""
import json
path = config.tasks_config_path(root)
label = config.tasks_config_label(root)
if not path.is_file():
if config.tasks_config_override() is not None:
raise ValidationError(
f"{config.ENV_TASKS_CONFIG} points at {path}, which is not a file. An override "
"that names nothing is never read as 'no tracker configured' - fix the path or "
"unset the variable."
)
"""The tracker configuration for `root`, or `None` when no `tasks` section
is set - which means no tracker is configured for this instance, not that
one failed to load. Raises `LegacySettingsLeftover` for an old
the tracker settings file of an earlier version, `ValidationError` for a broken setting."""
legacy_settings.refuse_leftover(root, "tasks")
loaded = settings.load(root, only=config_shapes.PROFILE_SECTIONS)
tasks = loaded.section("tasks")
if not tasks.configured:
return None
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
provider = tasks["provider"]
provider_section = loaded.section(provider)
if not provider_section.configured:
raise ValidationError(
f"{label} is unreadable ({exc}). It decides which task "
"tracker the weekly review talks to, so a broken file is not treated as 'no "
"tracker configured' - fix it or delete it deliberately."
) from exc
shape = config_shapes.TASKS
checked = config_shapes.check(shape, data)
if checked.problems:
raise ValidationError(
f"{label}: {checked.message()}. Expected: {config_shapes.render_expected(shape)}"
f"tasks.provider is {provider!r}, so the settings need a {provider} section - "
f"set its fields with `config set {provider}.<field> <value>`"
)
provider = checked.values["provider"]
thresholds = checked.values["thresholds"]
return TasksConfig(
provider=provider,
thresholds=Thresholds(**thresholds),
provider_config=data[provider],
thresholds=Thresholds(
stalled_waiting_days=tasks["stalled_waiting_days"],
unpaged_project_weeks=tasks["unpaged_project_weeks"],
someday_stale_months=tasks["someday_stale_months"],
),
provider_config=provider_section,
)
tools/chemenu/tasks/superproductivity.py
+38 -24
View File
@@ -125,7 +125,7 @@ from datetime import date, datetime, timezone
from pathlib import Path
from typing import Any, Optional
from chemenu import config_shapes
from chemenu import config_shapes, settings
from chemenu.errors import HumanInterventionRequired, ValidationError
from chemenu.tasks.protocol import (
OpenItem,
@@ -151,13 +151,16 @@ WAITING_TAG_TITLE = "waiting"
# this module ever writes it.
INBOX_PROJECT_ID = "INBOX_PROJECT"
# This section's shape - its fields per `access`, and these values - is the
# record `config_shapes.SUPERPRODUCTIVITY`.
# This section's fields and these values are the record
# `config_shapes.SUPERPRODUCTIVITY`.
DEFAULT_API_BASE_URL = config_shapes.SUPERPRODUCTIVITY.field("api_base_url").default
ACCESS_API = "api"
ACCESS_SNAPSHOT = "snapshot"
KNOWN_ACCESS = tuple(value for value, _ in config_shapes.SUPERPRODUCTIVITY.variants)
KNOWN_ACCESS = config_shapes.SUPERPRODUCTIVITY_ACCESS
_API_ONLY = ("api_base_url", "api_token")
_SNAPSHOT_ONLY = ("backups_dir",)
# `electron/backup.ts` writes exactly this shape - a fixed-width timestamp,
# no prefix. Restricting the glob to it (Gitea #133) is what keeps a manually
@@ -170,7 +173,7 @@ _SNAPSHOT_GLOB = (
@dataclass(frozen=True)
class SuperProductivityConfig:
"""This provider's own section of `.wikitool-tasks.json`
"""This provider's `superproductivity` section of the tiered settings
(`TasksConfig.provider_config`). `access` decides both halves at once -
which path is read *and* whether a write path exists at all - and the
section carries only the fields that access path uses (Gitea #133): a
@@ -183,26 +186,37 @@ class SuperProductivityConfig:
api_token: Optional[str]
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "SuperProductivityConfig":
shape = config_shapes.SUPERPRODUCTIVITY
checked = config_shapes.check(shape, data)
if checked.problems:
def from_section(cls, section: "settings.Section") -> "SuperProductivityConfig":
given = section.given
access = section["access"]
def reject(names: tuple[str, ...]) -> None:
present = [name for name in names if name in given]
if present:
raise ValidationError(
f"superproductivity config: access: '{access}' does not take {present} - a "
"section names only one access path's own fields"
)
if access == ACCESS_API:
reject(_SNAPSHOT_ONLY)
token = section["api_token"] if "api_token" in given else None
if not token:
raise ValidationError(
"superproductivity config: api_token is missing or empty, and required when "
"access: 'api' - every endpoint but GET /health requires "
"Authorization: Bearer <token>"
)
return cls(access=ACCESS_API, backups_dir=None,
api_base_url=section["api_base_url"], api_token=token)
reject(_API_ONLY)
if "backups_dir" not in given:
raise ValidationError(
f"superproductivity config: {checked.message()}. "
f"Expected one of: {config_shapes.render_expected(shape)}"
"superproductivity config: backups_dir is missing or empty, and required when "
"access: 'snapshot' - the read path has nothing to read otherwise"
)
values = checked.values
if values["access"] == ACCESS_API:
return cls(
access=ACCESS_API, backups_dir=None,
api_base_url=values["api_base_url"], api_token=values["api_token"],
)
return cls(
access=ACCESS_SNAPSHOT,
backups_dir=Path(values["backups_dir"]).expanduser(),
api_base_url=DEFAULT_API_BASE_URL,
api_token=None,
)
return cls(access=ACCESS_SNAPSHOT, backups_dir=Path(section["backups_dir"]).expanduser(),
api_base_url=DEFAULT_API_BASE_URL, api_token=None)
def latest_snapshot_path(cfg: SuperProductivityConfig) -> Path:
@@ -213,7 +227,7 @@ def latest_snapshot_path(cfg: SuperProductivityConfig) -> Path:
lexically *after* every timestamp and would otherwise pin every reader to
itself forever."""
directory = cfg.backups_dir
assert directory is not None # from_dict guarantees this for access: snapshot
assert directory is not None # from_section guarantees this for access: snapshot
if not directory.is_dir():
raise ValidationError(
f"superproductivity: backups_dir does not exist: {directory}"
tools/chemenu/telemetry/policy.py
+14 -7
View File
@@ -7,7 +7,8 @@ used to drift from the writer (it read `WIKI_TRACE` itself; fixed alongside
this module, see AGENTS.md invariant 8).
**Precedence for `enabled`:** `WIKI_TRACE` (either direction) beats
`.wikitool-telemetry.json`, which beats the installation-form default. The
the `telemetry` section of `.wikitool-local.json`, which beats the
installation-form default. The
form is read off `config.RELEASE_STAMP_FILENAME`: present means an exported/
distributed tree (default *off* - an operator never asked for telemetry),
absent means the dev checkout this package ships from (default *on* - the
@@ -15,11 +16,11 @@ traces are this stack's own measuring instrument, see
docs/explanation/telemetry-and-evaluation.md).
**Caching.** Only the filesystem-derived half - the release stamp and the
config file - is cached, keyed on the resolved root. `WIKI_TRACE` and the two
local settings file - is cached, keyed on the resolved root. `WIKI_TRACE` and the two
`WIKI_TRACE_MAX_SESSION_BYTES`/`WIKI_TRACE_KEEP_SESSIONS` overrides are read
fresh on every call: they are cheap (no I/O) and a test that flips one
mid-run must see the new value immediately, not a cached one. A test that
rewrites the release stamp or the config file at an already-resolved root
rewrites the release stamp or the local settings file at an already-resolved root
calls `reset_cache()` itself, the same discipline `conventions.reset_cache()`
already follows.
@@ -35,7 +36,7 @@ from pathlib import Path
from chemenu import config, config_shapes
# The file's shape - and these two defaults with it - is the record in
# The section's shape - and these two defaults with it - is the record in
# `chemenu.config_shapes`, which is stdlib-only for exactly this import.
DEFAULT_MAX_SESSION_BYTES = config_shapes.TELEMETRY.field("max_session_bytes").default
DEFAULT_KEEP_SESSIONS = config_shapes.TELEMETRY.field("keep_sessions").default
@@ -85,15 +86,21 @@ def _given(checked: "config_shapes.Checked", name: str) -> int | None:
def _read_config(root: Path) -> dict:
"""The `telemetry` section of `.wikitool-local.json`, read as leniently as
the hook needs: whatever cannot be read - the file, the JSON, the section -
is an absent section. Nothing else of the file is looked at, and the
telemetry file of an earlier version is not looked at at all: `doctor` and
`config migrate` deal with it, this hot path never fails on it."""
try:
raw = (root / config.TELEMETRY_FILENAME).read_text(encoding="utf-8")
raw = (root / config.LOCAL_SETTINGS_FILENAME).read_text(encoding="utf-8")
except OSError:
return {}
try:
data = json.loads(raw)
except json.JSONDecodeError:
return {}
return data if isinstance(data, dict) else {}
section = data.get(config_shapes.TELEMETRY.key) if isinstance(data, dict) else None
return section if isinstance(section, dict) else {}
def _root_defaults(root: Path) -> _RootDefaults:
@@ -114,7 +121,7 @@ def _root_defaults(root: Path) -> _RootDefaults:
checked = config_shapes.check(config_shapes.TELEMETRY, _read_config(root))
file_enabled = checked.values.get("enabled")
file_reason = (
f"{config.TELEMETRY_FILENAME} sets enabled={file_enabled}"
f"{config.LOCAL_SETTINGS_FILENAME} sets telemetry.enabled={file_enabled}"
if file_enabled is not None
else None
)
tools/chemenu/tests/conftest.py
+26 -1
View File
@@ -1,10 +1,11 @@
import json
import os
import subprocess
from pathlib import Path
import pytest
from chemenu import config, conventions, toolpaths
from chemenu import config, config_shapes, conventions, toolpaths
from chemenu.frontmatter_io import write_page
from chemenu.session import HARNESS_ENV_VARS
from chemenu.telemetry import policy as telemetry_policy
@@ -358,3 +359,27 @@ def kb_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
)
(kb / "log.md").write_text("# Wiki Log\n", encoding="utf-8")
return kb
def write_settings(root: Path, monkeypatch: pytest.MonkeyPatch, **sections: dict) -> None:
"""Put `sections` into the tiered settings files under `root`, each field
in the file its tier says. A plain value for a secret field is stood in by
a reference to `<section>_<field>` and the value goes into the environment
variable that reference reads - never into a file. Existing content is kept."""
files = {True: {"schema": 1, "instance_id": "testinstance"}, False: {"schema": 1}}
for in_instance, name in ((True, config.INSTANCE_SETTINGS_FILENAME), (False, config.LOCAL_SETTINGS_FILENAME)):
path = Path(root) / name
if path.is_file():
files[in_instance] = json.loads(path.read_text(encoding="utf-8"))
shapes = {shape.key: shape for shape in config_shapes.SETTINGS_SECTIONS}
for key, body in sections.items():
for field_name, value in body.items():
spec = shapes[key].field(field_name)
if spec.tier == config_shapes.SECRET and not (isinstance(value, dict) and "ref" in value):
ref = f"{key}_{field_name}"
monkeypatch.setenv(config.ENV_SECRET_PREFIX + ref.upper(), value)
value = {"ref": ref}
in_instance = spec.tier == config_shapes.INSTANCE
files[in_instance].setdefault(key, {})[field_name] = value
(Path(root) / config.INSTANCE_SETTINGS_FILENAME).write_text(json.dumps(files[True]), encoding="utf-8")
(Path(root) / config.LOCAL_SETTINGS_FILENAME).write_text(json.dumps(files[False]), encoding="utf-8")
tools/chemenu/tests/record_sp_fixtures.py
+2 -2
View File
@@ -34,8 +34,8 @@ def main(out: Path) -> None:
work = Path(tempfile.mkdtemp(prefix="chemenu-record-sp-"))
process = sp_headless.start(executable, work / "profile", marker_project=PROJECT)
try:
cfg = sp.SuperProductivityConfig.from_dict(
{"access": "api", "api_base_url": process.base_url, "api_token": process.token}
cfg = sp.SuperProductivityConfig(
access=sp.ACCESS_API, backups_dir=None, api_base_url=process.base_url, api_token=process.token
)
reader = sp.SuperProductivityApiReader(cfg)
writer = sp.SuperProductivityWriter(cfg, reader)
tools/chemenu/tests/test_bugreport.py
+2 -2
View File
@@ -57,7 +57,7 @@ def checkout(tmp_path: Path, monkeypatch) -> Path:
(root / "kb" / "Concepts" / "CONTRACT.md").write_text("stack file\n")
(root / "raw" / "articles" / "Quelle Gamma.md").write_text("source\n")
(root / "VERSION").write_text("1.2.3\n")
(root / ".wikitool-upload.json").write_text(
(root / ".wikitool-local.json").write_text(
json.dumps({"schema": 1, "api_token": TOKEN, "enabled": True})
)
(root / ".wikitool-tools.json").write_text(
@@ -133,7 +133,7 @@ def test_secrets_are_removed_everywhere_including_the_archive(checkout, tmp_path
assert TOKEN not in text
assert PASSWORD not in text
stack = json.loads((bundle / "stack.json").read_text())
assert stack["config_files"][".wikitool-upload.json"]["api_token"] == "<removed>"
assert stack["config_files"][".wikitool-local.json"]["api_token"] == "<removed>"
archive = bundle.with_name(bundle.name + ".zip")
with zipfile.ZipFile(archive) as zf:
for name in zf.namelist():
tools/chemenu/tests/test_caldav.py
+15 -10
View File
@@ -22,6 +22,7 @@ from xml.sax.saxutils import escape as _xml_escape
import icalendar
import pytest
from chemenu import config_shapes, settings
from chemenu.errors import ValidationError
from chemenu.tasks import caldav as cd
@@ -32,6 +33,10 @@ def _auth_header() -> str:
return "Basic " + base64.b64encode(f"{TOKEN[0]}:{TOKEN[1]}".encode()).decode("ascii")
def _from_dict(data) -> cd.CalDAVConfig:
return cd.CalDAVConfig.from_section(settings.Section.from_data(config_shapes.CALDAV, data))
def _cfg(port: int, *, path: str = "/cal/", **overrides) -> cd.CalDAVConfig:
data = {
"url": f"http://127.0.0.1:{port}{path}",
@@ -41,7 +46,7 @@ def _cfg(port: int, *, path: str = "/cal/", **overrides) -> cd.CalDAVConfig:
"someday_list": "Someday",
**overrides,
}
return cd.CalDAVConfig.from_dict(data)
return _from_dict(data)
# --- fixture builders --------------------------------------------------------------
@@ -338,18 +343,18 @@ def _fresh_state() -> dict:
return {"calendars": {}, "resources": {}, "created": []}
# --- CalDAVConfig.from_dict -----------------------------------------------------------
# --- CalDAVConfig.from_section -----------------------------------------------------------
def test_from_dict_requires_every_string_field():
with pytest.raises(ValidationError, match="url"):
cd.CalDAVConfig.from_dict({
_from_dict({
"username": "u", "app_password": "p", "inbox_list": "Inbox", "someday_list": "Someday",
})
def test_from_dict_rejects_unknown_fields():
with pytest.raises(ValidationError, match="does not take"):
cd.CalDAVConfig.from_dict({
with pytest.raises(ValidationError, match="not a field"):
_from_dict({
"url": "https://x/", "username": "u", "app_password": "p",
"inbox_list": "Inbox", "someday_list": "Someday", "backups_dir": "/x",
})
@@ -357,14 +362,14 @@ def test_from_dict_rejects_unknown_fields():
def test_from_dict_exclude_lists_must_be_a_list_of_strings():
with pytest.raises(ValidationError, match="exclude_lists"):
cd.CalDAVConfig.from_dict({
_from_dict({
"url": "https://x/", "username": "u", "app_password": "p",
"inbox_list": "Inbox", "someday_list": "Someday", "exclude_lists": "not-a-list",
})
def test_from_dict_exclude_lists_defaults_to_empty():
cfg = cd.CalDAVConfig.from_dict({
cfg = _from_dict({
"url": "https://x/", "username": "u", "app_password": "p",
"inbox_list": "Inbox", "someday_list": "Someday",
})
@@ -372,7 +377,7 @@ def test_from_dict_exclude_lists_defaults_to_empty():
def test_from_dict_normalizes_a_missing_trailing_slash():
cfg = cd.CalDAVConfig.from_dict({
cfg = _from_dict({
"url": "https://x/cal", "username": "u", "app_password": "p",
"inbox_list": "Inbox", "someday_list": "Someday",
})
@@ -588,7 +593,7 @@ def test_unparsable_vtodo_fails_loud_not_silent():
def test_unreachable_server_fails_loud():
cfg = cd.CalDAVConfig.from_dict({
cfg = _from_dict({
"url": "http://127.0.0.1:1/cal/", "username": "u", "app_password": "p",
"inbox_list": "Inbox", "someday_list": "Someday",
})
@@ -613,7 +618,7 @@ def test_probe_reports_authentication_failure():
def test_probe_reports_unreachable():
cfg = cd.CalDAVConfig.from_dict({
cfg = _from_dict({
"url": "http://127.0.0.1:1/cal/", "username": "u", "app_password": "p",
"inbox_list": "Inbox", "someday_list": "Someday",
})
tools/chemenu/tests/test_config_shapes.py
+78 -379
View File
@@ -1,17 +1,13 @@
"""`chemenu.config_shapes` - one record per hand-written configuration file,
"""`chemenu.config_shapes` - one record per section of the tiered settings,
which the loaders read and `docs reference` renders.
The drop-in half compares every loader against a copy of the loader as it stood
before the records existed (the `_legacy_*` oracles below, kept verbatim in
what they accept): any file it accepted must still be accepted with the same
values, and any file it rejected must still be rejected. Only the wording of a
rejection may differ.
The loaders take their field names, required fields, defaults, coercions and
tiers from the record: a test changes the record and the loader follows.
"""
from __future__ import annotations
import ast
import dataclasses
import json
import re
import sys
from pathlib import Path
@@ -27,11 +23,29 @@ from chemenu.errors import ValidationError
from chemenu.tasks import caldav, superproductivity
from chemenu.tasks import config as tasks_config
from chemenu.telemetry import policy
from chemenu.tests.conftest import write_settings
runner = CliRunner()
PACKAGE = Path(config_shapes.__file__).parent
UPLOAD_OK = {
"identity_header": "X-User", "max_bytes": 1000, "allowed_extensions": [".md", ".PDF"],
"quota_submissions_per_day": 5, "quota_bytes_per_day": 5000,
}
TASKS_OK = {"provider": "caldav", "stalled_waiting_days": 14, "unpaged_project_weeks": 3,
"someday_stale_months": 5}
CALDAV_OK = {"url": "https://dav.example/", "username": "u", "app_password": "pw",
"inbox_list": "Inbox", "someday_list": "Someday"}
@pytest.fixture
def root(tmp_path, monkeypatch):
tree = tmp_path / "instance"
tree.mkdir()
monkeypatch.setattr(config, "ROOT", tree)
return tree
# --- the module itself -------------------------------------------------------
@@ -62,10 +76,9 @@ def test_the_telemetry_policy_reads_the_record():
assert policy.DEFAULT_KEEP_SESSIONS == config_shapes.TELEMETRY.field("keep_sessions").default
def test_one_record_per_file():
assert [shape.key for shape in config_shapes.SHAPES] == [
config.TELEMETRY_FILENAME, config.PUBLISH_REMOTES_FILENAME,
config.UPLOAD_CONFIG_FILENAME, config.TASKS_CONFIG_FILENAME,
def test_one_record_per_section():
assert [shape.key for shape in config_shapes.SETTINGS_SECTIONS] == [
"telemetry", "remotes", "upload", "tasks", "superproductivity", "caldav",
]
@@ -87,319 +100,12 @@ def test_the_known_names_follow_the_records():
assert superproductivity.KNOWN_ACCESS == ("api", "snapshot")
# --- the legacy oracles: the loaders as they were before the records --------
def _legacy_upload(data):
if not isinstance(data, dict):
raise ValidationError("not an object")
try:
identity_header = str(data.get("identity_header") or "X-Forwarded-User")
max_bytes = int(data["max_bytes"])
extensions = tuple(sorted({str(ext).lower() for ext in data["allowed_extensions"]}))
quota = data["quota"]
submissions_per_day = int(quota["submissions_per_day"])
bytes_per_day = int(quota["bytes_per_day"])
except (KeyError, TypeError, ValueError) as exc:
raise ValidationError("misshaped") from exc
if max_bytes <= 0 or submissions_per_day <= 0 or bytes_per_day <= 0:
raise ValidationError("not positive")
if not extensions or [ext for ext in extensions if not ext.startswith(".")]:
raise ValidationError("extensions")
return (identity_header, max_bytes, extensions, submissions_per_day, bytes_per_day)
def _legacy_tasks(data):
if not isinstance(data, dict):
raise ValidationError("not an object")
try:
provider = str(data["provider"])
raw = data["thresholds"]
thresholds = (int(raw["stalled_waiting_days"]), int(raw["unpaged_project_weeks"]),
int(raw["someday_stale_months"]))
except (KeyError, TypeError, ValueError) as exc:
raise ValidationError("misshaped") from exc
if provider not in ("superproductivity", "caldav"):
raise ValidationError("provider")
if any(value <= 0 for value in thresholds):
raise ValidationError("not positive")
if not isinstance(data.get(provider), dict):
raise ValidationError("section")
return (provider, thresholds, data.get(provider))
def _legacy_sp(data):
if not isinstance(data, dict):
raise ValidationError("not an object")
access = data.get("access")
if access not in ("api", "snapshot"):
raise ValidationError("access")
if access == "api":
if set(data) - {"access", "api_base_url", "api_token"}:
raise ValidationError("extra")
token = data.get("api_token")
if not isinstance(token, str) or not token:
raise ValidationError("token")
return ("api", None, str(data.get("api_base_url") or "http://127.0.0.1:3876"), token)
if set(data) - {"access", "backups_dir"}:
raise ValidationError("extra")
if not data.get("backups_dir"):
raise ValidationError("backups_dir")
return ("snapshot", Path(data["backups_dir"]).expanduser(), "http://127.0.0.1:3876", None)
_CALDAV_STRINGS = ("url", "username", "app_password", "inbox_list", "someday_list")
def _legacy_caldav(data):
if not isinstance(data, dict):
raise ValidationError("not an object")
if [k for k in _CALDAV_STRINGS if not isinstance(data.get(k), str) or not data[k]]:
raise ValidationError("missing")
if set(data) - {*_CALDAV_STRINGS, "exclude_lists"}:
raise ValidationError("extra")
exclude = data.get("exclude_lists", [])
if not isinstance(exclude, list) or not all(isinstance(x, str) for x in exclude):
raise ValidationError("exclude_lists")
url = data["url"] if data["url"].endswith("/") else data["url"] + "/"
return (url, data["username"], data["app_password"], data["inbox_list"], data["someday_list"],
tuple(exclude))
def _legacy_remotes(data):
urls = data.get("allowed_push_urls")
if not isinstance(urls, list) or not all(isinstance(u, str) for u in urls):
raise ValidationError("urls")
return urls
def _legacy_telemetry(data):
def positive(value):
try:
parsed = int(value)
except (TypeError, ValueError):
return None
return parsed if parsed > 0 else None
data = data if isinstance(data, dict) else {}
return (
bool(data["enabled"]) if "enabled" in data else None,
positive(data.get("max_session_bytes")),
positive(data.get("keep_sessions")),
)
# --- the loaders today, reduced to the same tuples --------------------------
def _upload(data, tmp_path):
(tmp_path / config.UPLOAD_CONFIG_FILENAME).write_text(json.dumps(data), encoding="utf-8")
from chemenu import upload
cfg = upload.read_config(tmp_path)
return (cfg.identity_header, cfg.max_bytes, cfg.allowed_extensions, cfg.submissions_per_day,
cfg.bytes_per_day)
def _tasks(data, tmp_path):
(tmp_path / config.TASKS_CONFIG_FILENAME).write_text(json.dumps(data), encoding="utf-8")
cfg = tasks_config.read_config(tmp_path)
t = cfg.thresholds
return (cfg.provider, (t.stalled_waiting_days, t.unpaged_project_weeks, t.someday_stale_months),
cfg.provider_config)
def _sp(data, tmp_path):
cfg = superproductivity.SuperProductivityConfig.from_dict(data)
return (cfg.access, cfg.backups_dir, cfg.api_base_url, cfg.api_token)
def _caldav(data, tmp_path):
cfg = caldav.CalDAVConfig.from_dict(data)
return (cfg.url, cfg.username, cfg.app_password, cfg.inbox_list, cfg.someday_list,
cfg.exclude_lists)
def _remotes(data, tmp_path):
(tmp_path / config.PUBLISH_REMOTES_FILENAME).write_text(json.dumps(data), encoding="utf-8")
try:
return git_publish.read_allowed_push_urls()
except typer.Exit as exc:
raise ValidationError("fail()") from exc
def _telemetry(data, tmp_path):
(tmp_path / config.TELEMETRY_FILENAME).write_text(json.dumps(data), encoding="utf-8")
policy.reset_cache()
defaults = policy._root_defaults(tmp_path.resolve())
policy.reset_cache()
return (defaults.file_enabled, defaults.file_max_bytes, defaults.file_keep_sessions)
UPLOAD_OK = {
"schema": 1, "identity_header": "X-Forwarded-User", "max_bytes": 10485760,
"allowed_extensions": [".md", ".txt", ".pdf", ".html", ".csv", ".json", ".png", ".jpg"],
"quota": {"submissions_per_day": 20, "bytes_per_day": 52428800},
}
TASKS_OK = {
"schema": 1, "provider": "superproductivity",
"thresholds": {"stalled_waiting_days": 14, "unpaged_project_weeks": 3, "someday_stale_months": 5},
"superproductivity": {"access": "api", "api_base_url": "http://127.0.0.1:3876", "api_token": "t"},
}
CALDAV_OK = {
"url": "https://host/remote.php/dav/calendars/me", "username": "me", "app_password": "pw",
"inbox_list": "Inbox", "someday_list": "Someday", "exclude_lists": ["Shopping"],
}
def _with(base, **changes):
data = json.loads(json.dumps(base))
for key, value in changes.items():
target = data
*parents, leaf = key.split("__")
for parent in parents:
target = target[parent]
if value is _DROP:
del target[leaf]
else:
target[leaf] = value
return data
_DROP = object()
CASES = [
# upload: the documented file, then every tolerance and every rejection
("upload", UPLOAD_OK),
("upload", _with(UPLOAD_OK, schema=_DROP)),
("upload", _with(UPLOAD_OK, identity_header=_DROP)),
("upload", _with(UPLOAD_OK, identity_header="")),
("upload", _with(UPLOAD_OK, max_bytes="10485760")),
("upload", _with(UPLOAD_OK, quota__bytes_per_day="100")),
("upload", _with(UPLOAD_OK, allowed_extensions=[".MD", ".md", ".Pdf"])),
("upload", _with(UPLOAD_OK, surplus="ignored")),
("upload", _with(UPLOAD_OK, quota__surplus=1)),
("upload", _with(UPLOAD_OK, max_bytes=0)),
("upload", _with(UPLOAD_OK, max_bytes="ten")),
("upload", _with(UPLOAD_OK, max_bytes=_DROP)),
("upload", _with(UPLOAD_OK, allowed_extensions=["md"])),
("upload", _with(UPLOAD_OK, allowed_extensions=[])),
("upload", _with(UPLOAD_OK, allowed_extensions=".md")),
("upload", _with(UPLOAD_OK, quota=[1, 2])),
("upload", _with(UPLOAD_OK, quota__submissions_per_day=-1)),
("upload", [1, 2]),
# tasks
("tasks", TASKS_OK),
("tasks", _with(TASKS_OK, schema=_DROP)),
("tasks", _with(TASKS_OK, thresholds__stalled_waiting_days="14")),
("tasks", _with(TASKS_OK, caldav={"url": "unused"})),
("tasks", _with(TASKS_OK, surplus=True)),
("tasks", _with(TASKS_OK, thresholds__surplus=1)),
("tasks", _with(TASKS_OK, provider="asana")),
("tasks", _with(TASKS_OK, provider=_DROP)),
("tasks", _with(TASKS_OK, thresholds__someday_stale_months=0)),
("tasks", _with(TASKS_OK, thresholds__unpaged_project_weeks=_DROP)),
("tasks", _with(TASKS_OK, thresholds="soon")),
("tasks", _with(TASKS_OK, superproductivity=_DROP)),
("tasks", _with(TASKS_OK, superproductivity="api")),
("tasks", _with(TASKS_OK, provider="caldav")),
("tasks", "not an object"),
# the Super Productivity section
("sp", {"access": "api", "api_base_url": "http://127.0.0.1:3876", "api_token": "t"}),
("sp", {"access": "api", "api_token": "t"}),
("sp", {"access": "api", "api_token": "t", "api_base_url": ""}),
("sp", {"access": "snapshot", "backups_dir": "~/.config/superProductivity/backups"}),
("sp", {"access": "api"}),
("sp", {"access": "api", "api_token": ""}),
("sp", {"access": "api", "api_token": 7}),
("sp", {"access": "api", "api_token": "t", "backups_dir": "/x"}),
("sp", {"access": "snapshot", "backups_dir": "/x", "api_token": "t"}),
("sp", {"access": "snapshot"}),
("sp", {"access": "snapshot", "backups_dir": ""}),
("sp", {"access": "auto", "backups_dir": "/x"}),
("sp", {"backups_dir": "/x"}),
("sp", ["api"]),
# the CalDAV section
("caldav", CALDAV_OK),
("caldav", _with(CALDAV_OK, exclude_lists=_DROP)),
("caldav", _with(CALDAV_OK, url="https://host/dav/")),
("caldav", _with(CALDAV_OK, url=_DROP)),
("caldav", _with(CALDAV_OK, username="")),
("caldav", _with(CALDAV_OK, app_password=5)),
("caldav", _with(CALDAV_OK, exclude_lists="Shopping")),
("caldav", _with(CALDAV_OK, exclude_lists=[1])),
("caldav", _with(CALDAV_OK, backups_dir="/x")),
# remotes
("remotes", {"schema": 1, "allowed_push_urls": ["git@example.org:me/my-wiki.git"]}),
("remotes", {"allowed_push_urls": []}),
("remotes", {"allowed_push_urls": ["a"], "surplus": 1}),
("remotes", {"allowed_push_urls": "not-a-list"}),
("remotes", {"allowed_push_urls": [1]}),
("remotes", {}),
# telemetry: never rejected, only read differently
("telemetry", {"enabled": True, "max_session_bytes": 5242880, "keep_sessions": 250}),
("telemetry", {}),
("telemetry", {"enabled": "no"}),
("telemetry", {"enabled": 0}),
("telemetry", {"enabled": None}),
("telemetry", {"max_session_bytes": "100", "keep_sessions": 3.7}),
("telemetry", {"max_session_bytes": 0, "keep_sessions": -2}),
("telemetry", {"max_session_bytes": "lots", "keep_sessions": [1]}),
("telemetry", {"surplus": 1, "schema": 1}),
("telemetry", [True]),
]
ORACLES = {
"upload": (_legacy_upload, _upload),
"tasks": (_legacy_tasks, _tasks),
"sp": (_legacy_sp, _sp),
"caldav": (_legacy_caldav, _caldav),
"remotes": (_legacy_remotes, _remotes),
"telemetry": (_legacy_telemetry, _telemetry),
}
def _outcome(function, data, *args):
try:
return ("accepted", function(data, *args))
except ValidationError:
return ("rejected", None)
@pytest.mark.parametrize("loader,data", CASES, ids=[f"{n}-{i}" for i, (n, _) in enumerate(CASES)])
def test_every_loader_accepts_and_rejects_exactly_what_it_did_before(loader, data, tmp_path, monkeypatch):
legacy, current = ORACLES[loader]
monkeypatch.setattr(config, "ROOT", tmp_path)
assert _outcome(current, data, tmp_path) == _outcome(legacy, data)
def test_the_cases_cover_both_outcomes_for_every_loader(tmp_path):
for loader, (legacy, _) in ORACLES.items():
if loader == "telemetry":
continue
outcomes = {_outcome(legacy, data)[0] for name, data in CASES if name == loader}
assert outcomes == {"accepted", "rejected"}, loader
def test_a_rejection_keeps_its_cause_and_renders_the_expected_text(tmp_path):
(tmp_path / config.UPLOAD_CONFIG_FILENAME).write_text(
json.dumps(_with(UPLOAD_OK, max_bytes=0)), encoding="utf-8"
)
from chemenu import upload
with pytest.raises(ValidationError) as caught:
upload.read_config(tmp_path)
message = str(caught.value)
assert "max_bytes must be positive" in message
assert f"Expected: {config_shapes.render_expected(config_shapes.UPLOAD)}" in message
# --- the loaders take their expectations from the record --------------------
def _add_required(shape, name="added"):
def _add_required(shape, name="added", tier=config_shapes.CHECKOUT):
return dataclasses.replace(
shape, fields=shape.fields + (config_shapes.Field(name, "string", "x", required=True),)
shape, fields=shape.fields + (config_shapes.Field(name, "string", "x", required=True, tier=tier),)
)
@@ -411,78 +117,78 @@ def _not_required(shape, name, default):
return dataclasses.replace(shape, fields=fields)
def test_upload_follows_a_field_made_required_or_optional(tmp_path, monkeypatch):
def _swap(monkeypatch, attr, shape):
monkeypatch.setattr(config_shapes, attr, shape)
monkeypatch.setattr(
config_shapes, "SETTINGS_SECTIONS",
tuple(shape if s.key == shape.key else s for s in config_shapes.SETTINGS_SECTIONS),
)
def test_upload_follows_a_field_made_required_or_optional(root, monkeypatch):
from chemenu import upload
(tmp_path / config.UPLOAD_CONFIG_FILENAME).write_text(json.dumps(UPLOAD_OK), encoding="utf-8")
monkeypatch.setattr(config_shapes, "UPLOAD", _add_required(config_shapes.UPLOAD))
original = config_shapes.UPLOAD
write_settings(root, monkeypatch, upload=UPLOAD_OK)
assert upload.read_config(root).max_bytes == 1000
_swap(monkeypatch, "UPLOAD", _add_required(original))
with pytest.raises(ValidationError, match="added is missing"):
upload.read_config(tmp_path)
upload.read_config(root)
monkeypatch.undo()
(tmp_path / config.UPLOAD_CONFIG_FILENAME).write_text(
json.dumps(_with(UPLOAD_OK, max_bytes=_DROP)), encoding="utf-8"
)
_swap(monkeypatch, "UPLOAD", original)
(root / config.LOCAL_SETTINGS_FILENAME).unlink()
(root / config.INSTANCE_SETTINGS_FILENAME).unlink()
write_settings(root, monkeypatch, upload={k: v for k, v in UPLOAD_OK.items() if k != "max_bytes"})
with pytest.raises(ValidationError):
upload.read_config(tmp_path)
monkeypatch.setattr(config_shapes, "UPLOAD", _not_required(config_shapes.UPLOAD, "max_bytes", 99))
assert upload.read_config(tmp_path).max_bytes == 99
upload.read_config(root)
_swap(monkeypatch, "UPLOAD", _not_required(original, "max_bytes", 99))
assert upload.read_config(root).max_bytes == 99
def test_tasks_follows_a_field_made_required(tmp_path, monkeypatch):
(tmp_path / config.TASKS_CONFIG_FILENAME).write_text(json.dumps(TASKS_OK), encoding="utf-8")
assert tasks_config.read_config(tmp_path) is not None
monkeypatch.setattr(config_shapes, "TASKS", _add_required(config_shapes.TASKS))
def test_tasks_follows_a_field_made_required(root, monkeypatch):
write_settings(root, monkeypatch, tasks=TASKS_OK, caldav=CALDAV_OK)
assert tasks_config.read_config(root) is not None
_swap(monkeypatch, "TASKS", _add_required(config_shapes.TASKS, tier=config_shapes.INSTANCE))
with pytest.raises(ValidationError, match="added is missing"):
tasks_config.read_config(tmp_path)
tasks_config.read_config(root)
def test_remotes_follows_a_field_made_required(tmp_path, monkeypatch):
monkeypatch.setattr(config, "ROOT", tmp_path)
(tmp_path / config.PUBLISH_REMOTES_FILENAME).write_text(
json.dumps({"allowed_push_urls": ["u"]}), encoding="utf-8"
)
def test_remotes_follows_a_field_made_required(root, monkeypatch):
write_settings(root, monkeypatch, remotes={"allowed_push_urls": ["u"]})
assert git_publish.read_allowed_push_urls() == ["u"]
monkeypatch.setattr(config_shapes, "REMOTES", _add_required(config_shapes.REMOTES))
_swap(monkeypatch, "REMOTES", _add_required(config_shapes.REMOTES))
with pytest.raises(typer.Exit):
git_publish.read_allowed_push_urls()
def test_caldav_follows_a_field_made_required(monkeypatch):
data = _with(CALDAV_OK, exclude_lists=_DROP)
assert caldav.CalDAVConfig.from_dict(data).exclude_lists == ()
from chemenu import settings
monkeypatch.setenv("WIKITOOL_SECRET_X", "pw")
data = dict(CALDAV_OK, app_password={"ref": "x"})
assert caldav.CalDAVConfig.from_section(
settings.Section.from_data(config_shapes.CALDAV, data)).exclude_lists == ()
fields = tuple(
dataclasses.replace(f, required=True) if f.name == "exclude_lists" else f
for f in config_shapes.CALDAV.fields
)
monkeypatch.setattr(config_shapes, "CALDAV", dataclasses.replace(config_shapes.CALDAV, fields=fields))
shape = dataclasses.replace(config_shapes.CALDAV, fields=fields)
with pytest.raises(ValidationError, match="exclude_lists is missing"):
caldav.CalDAVConfig.from_dict(data)
settings.Section.from_data(shape, data)
def test_superproductivity_follows_a_field_added_to_a_variant(monkeypatch):
data = {"access": "api", "api_token": "t", "timeout": "5"}
with pytest.raises(ValidationError, match="does not take"):
superproductivity.SuperProductivityConfig.from_dict(data)
shape = config_shapes.SUPERPRODUCTIVITY
api = dict(shape.variants)["api"] + (config_shapes.Field("timeout", "string", "x"),)
variants = (("api", api),) + tuple(v for v in shape.variants if v[0] != "api")
monkeypatch.setattr(config_shapes, "SUPERPRODUCTIVITY", dataclasses.replace(shape, variants=variants))
assert superproductivity.SuperProductivityConfig.from_dict(data).api_token == "t"
def test_telemetry_follows_a_condition_in_the_record(tmp_path, monkeypatch):
(tmp_path / config.TELEMETRY_FILENAME).write_text(json.dumps({"keep_sessions": 500}), encoding="utf-8")
def test_telemetry_follows_a_condition_in_the_record(root, monkeypatch):
write_settings(root, monkeypatch, telemetry={"keep_sessions": 500})
policy.reset_cache()
assert policy.resolve(tmp_path).keep_sessions == 500
assert policy.resolve(root).keep_sessions == 500
shape = config_shapes.TELEMETRY
fields = tuple(
dataclasses.replace(f, check=lambda v: None if v < 100 else "too many") if f.name == "keep_sessions" else f
for f in shape.fields
)
monkeypatch.setattr(config_shapes, "TELEMETRY", dataclasses.replace(shape, fields=fields))
_swap(monkeypatch, "TELEMETRY", dataclasses.replace(shape, fields=fields))
policy.reset_cache()
assert policy.resolve(tmp_path).keep_sessions == policy.DEFAULT_KEEP_SESSIONS
assert policy.resolve(root).keep_sessions == policy.DEFAULT_KEEP_SESSIONS
policy.reset_cache()
@@ -505,16 +211,15 @@ def test_a_telemetry_deviation_names_its_field(data, field):
def test_a_conforming_telemetry_file_has_no_deviation():
for example in config_shapes.TELEMETRY.examples:
assert config_shapes.deviations(config_shapes.TELEMETRY, example) == []
assert config_shapes.deviations(config_shapes.TELEMETRY, {"schema": 1, "enabled": False}) == []
assert config_shapes.deviations(config_shapes.TELEMETRY, {"enabled": False}) == []
def test_every_example_passes_its_own_record():
for shape in config_shapes.SHAPES:
from chemenu import settings
for shape in config_shapes.SETTINGS_SECTIONS:
for example in shape.examples:
assert config_shapes.check(shape, example).problems == [], shape.key
for section in config_shapes.TASKS.sections:
for example in section.examples:
assert config_shapes.check(section, example).problems == [], section.key
settings.Section.from_data(shape, example)
# --- the generated regions --------------------------------------------------
@@ -524,10 +229,9 @@ def _page() -> str:
return (config.ROOT / reference_doc.CONFIGURATION).read_text(encoding="utf-8")
def _all_fields(shape, prefix=""):
for spec in shape.fields + tuple(f for _, fs in shape.variants for f in fs):
yield prefix + spec.name
yield from (prefix + spec.name + "." + sub.name for sub in spec.fields)
def _all_fields(shape):
for spec in shape.fields:
yield spec.name
def test_the_page_carries_one_region_per_file_and_every_field_in_it():
@@ -538,17 +242,12 @@ def test_the_page_carries_one_region_per_file_and_every_field_in_it():
assert region is not None, name
for path in _all_fields(shape):
assert f"`{path}`" in region, (name, path)
for section in shape.sections:
for path in _all_fields(section):
assert f"`{path}`" in region, (name, path)
@pytest.mark.parametrize("relative", [reference_doc.CONFIGURATION, "docs/how-to/set-up-the-mcp-server.md"])
def test_no_json_example_for_these_files_stands_outside_a_region(relative):
text = blocks.mask_regions((config.ROOT / relative).read_text(encoding="utf-8"))
keys = {f.name for shape in config_shapes.SHAPES for f in shape.fields if f.name != "schema"}
keys |= {f.name for s in config_shapes.TASKS.sections for f in s.fields}
keys |= {f.name for s in config_shapes.TASKS.sections for _, fs in s.variants for f in fs}
keys = {f.name for shape in config_shapes.SETTINGS_SECTIONS for f in shape.fields}
for block in re.findall(r"```json\n(.*?)```", text, re.S):
assert not any(f'"{key}"' in block for key in keys), block
@@ -596,7 +295,7 @@ def test_apply_fills_every_region_of_the_page_and_a_second_run_changes_nothing(t
def test_a_changed_record_without_apply_fails_docs_verify(monkeypatch):
assert not [i for i in docs_verify.check_reference_regions() if "configuration.md" in i]
monkeypatch.setattr(config_shapes, "REMOTES", _add_required(config_shapes.REMOTES))
_swap(monkeypatch, "REMOTES", _add_required(config_shapes.REMOTES))
issues = [i for i in docs_verify.check_reference_regions() if reference_doc.CONFIGURATION in i]
assert len(issues) == 1
assert "reference-config-remotes" in issues[0] and "docs reference --apply" in issues[0]
tools/chemenu/tests/test_doctor.py
+74 -88
View File
@@ -13,6 +13,7 @@ import pytest
from chemenu import config, conventions, prerequisites, toolpaths
from chemenu.commands import doctor, instructions_cmd
from chemenu.tests.conftest import write_settings
def _git(root: Path, *args: str) -> None:
@@ -311,25 +312,26 @@ def test_a_renamed_but_unfilled_environment_template_warns(instance):
assert "template" in next(c.detail for c in checks if c.name == "environment")
_THRESHOLDS = {"stalled_waiting_days": 14, "unpaged_project_weeks": 3, "someday_stale_months": 5}
def _remotes_detail(checks) -> str:
return next(c.detail for c in checks if c.name == "publish-remotes")
def test_publish_remotes_says_not_armed_when_the_file_is_absent(instance):
def test_publish_remotes_says_not_armed_when_the_section_is_absent(instance):
"""Absence stays OK - a single-remote checkout has nothing to protect - but
the line has to say the gate is off. AGENTS.md lists it among the limits
enforced in code, so "no .wikitool-remotes.json" alone leaves a reader
enforced in code, so "no remotes section" alone leaves a reader
trusting a safeguard that is not running."""
checks = doctor.run_doctor()
assert _status(checks, "publish-remotes") == "OK"
assert "not armed" in _remotes_detail(checks)
def test_publish_remotes_says_armed_when_the_file_lists_a_target(instance):
(config.ROOT / config.PUBLISH_REMOTES_FILENAME).write_text(
'{ "schema": 1, "allowed_push_urls": ["ssh://git@example.net/one.git"] }',
encoding="utf-8",
)
def test_publish_remotes_says_armed_when_the_section_lists_a_target(instance, monkeypatch):
write_settings(config.ROOT, monkeypatch,
remotes={"allowed_push_urls": ["ssh://git@example.net/one.git"]})
checks = doctor.run_doctor()
assert _status(checks, "publish-remotes") == "OK"
detail = _remotes_detail(checks)
@@ -345,18 +347,17 @@ def test_publish_remotes_warns_on_several_remotes_without_an_allowlist(instance)
assert "not armed" in _remotes_detail(checks)
def test_upload_intake_ok_when_no_config_file(instance):
def test_upload_intake_ok_when_no_section(instance):
checks = doctor.run_doctor()
assert _status(checks, "upload-intake") == "OK"
assert "not registered" in _detail(checks, "upload-intake")
def test_upload_intake_ok_and_armed_when_config_is_valid(instance):
(config.ROOT / config.UPLOAD_CONFIG_FILENAME).write_text(
'{"schema": 1, "max_bytes": 100, "allowed_extensions": [".md"], '
'"quota": {"submissions_per_day": 1, "bytes_per_day": 100}}',
encoding="utf-8",
)
def test_upload_intake_ok_and_armed_when_section_is_valid(instance, monkeypatch):
write_settings(config.ROOT, monkeypatch, upload={
"max_bytes": 100, "allowed_extensions": [".md"],
"quota_submissions_per_day": 1, "quota_bytes_per_day": 100,
})
checks = doctor.run_doctor()
assert _status(checks, "upload-intake") == "OK"
detail = _detail(checks, "upload-intake")
@@ -364,50 +365,37 @@ def test_upload_intake_ok_and_armed_when_config_is_valid(instance):
assert "0 submission(s) waiting" in detail
def test_upload_intake_fails_on_a_malformed_config(instance):
(config.ROOT / config.UPLOAD_CONFIG_FILENAME).write_text("{not json", encoding="utf-8")
def test_upload_intake_fails_on_a_malformed_section(instance, monkeypatch):
write_settings(config.ROOT, monkeypatch, upload={"max_bytes": "many"})
checks = doctor.run_doctor()
assert _status(checks, "upload-intake") == "FAIL"
def test_tasks_provider_ok_when_no_config_file(instance):
def test_tasks_provider_ok_when_no_section(instance):
checks = doctor.run_doctor()
assert _status(checks, "tasks-provider") == "OK"
assert "no task tracker configured" in _detail(checks, "tasks-provider")
def test_tasks_provider_fails_on_a_malformed_config(instance):
(config.ROOT / config.TASKS_CONFIG_FILENAME).write_text("{not json", encoding="utf-8")
def test_tasks_provider_fails_on_a_malformed_settings_file(instance):
(config.ROOT / config.LOCAL_SETTINGS_FILENAME).write_text("{not json", encoding="utf-8")
checks = doctor.run_doctor()
assert _status(checks, "tasks-provider") == "FAIL"
def test_tasks_provider_fails_on_a_bad_superproductivity_section(instance):
(config.ROOT / config.TASKS_CONFIG_FILENAME).write_text(
'{"schema": 1, "provider": "superproductivity", "thresholds": '
'{"stalled_waiting_days": 14, "unpaged_project_weeks": 3, "someday_stale_months": 5}, '
'"superproductivity": {}}',
encoding="utf-8",
)
def test_tasks_provider_fails_on_a_bad_superproductivity_section(instance, monkeypatch):
write_settings(config.ROOT, monkeypatch, tasks={"provider": "superproductivity", **_THRESHOLDS})
checks = doctor.run_doctor()
assert _status(checks, "tasks-provider") == "FAIL"
def test_tasks_provider_ok_but_names_the_unready_read_path_when_configured(instance):
def test_tasks_provider_ok_but_names_the_unready_read_path_when_configured(instance, monkeypatch):
"""`access: snapshot` - only the read path is a finding; the API is not
even attempted (Gitea #133)."""
backups = config.ROOT / "sp-backups"
(config.ROOT / config.TASKS_CONFIG_FILENAME).write_text(
json.dumps({
"schema": 1, "provider": "superproductivity",
"thresholds": {
"stalled_waiting_days": 14, "unpaged_project_weeks": 3,
"someday_stale_months": 5,
},
"superproductivity": {"access": "snapshot", "backups_dir": str(backups)},
}),
encoding="utf-8",
)
write_settings(config.ROOT, monkeypatch,
tasks={"provider": "superproductivity", **_THRESHOLDS},
superproductivity={"access": "snapshot", "backups_dir": str(backups)})
checks = doctor.run_doctor()
assert _status(checks, "tasks-provider") == "OK"
detail = _detail(checks, "tasks-provider")
@@ -415,20 +403,13 @@ def test_tasks_provider_ok_but_names_the_unready_read_path_when_configured(insta
assert "not ready" in detail
def test_tasks_provider_ok_but_names_api_unreachable_when_configured(instance):
def test_tasks_provider_ok_but_names_api_unreachable_when_configured(instance, monkeypatch):
"""`access: api` - the API reachability is the finding; the (irrelevant)
backup file is not even looked at (Gitea #133)."""
(config.ROOT / config.TASKS_CONFIG_FILENAME).write_text(
json.dumps({
"schema": 1, "provider": "superproductivity",
"thresholds": {
"stalled_waiting_days": 14, "unpaged_project_weeks": 3,
"someday_stale_months": 5,
},
"superproductivity": {"access": "api", "api_base_url": "http://127.0.0.1:1", "api_token": "t"},
}),
encoding="utf-8",
)
write_settings(config.ROOT, monkeypatch,
tasks={"provider": "superproductivity", **_THRESHOLDS},
superproductivity={"access": "api", "api_base_url": "http://127.0.0.1:1",
"api_token": "t"})
checks = doctor.run_doctor()
assert _status(checks, "tasks-provider") == "OK"
detail = _detail(checks, "tasks-provider")
@@ -436,32 +417,17 @@ def test_tasks_provider_ok_but_names_api_unreachable_when_configured(instance):
assert "not reachable" in detail
def test_tasks_provider_fails_on_a_bad_caldav_section(instance):
(config.ROOT / config.TASKS_CONFIG_FILENAME).write_text(
'{"schema": 1, "provider": "caldav", "thresholds": '
'{"stalled_waiting_days": 14, "unpaged_project_weeks": 3, "someday_stale_months": 5}, '
'"caldav": {}}',
encoding="utf-8",
)
def test_tasks_provider_fails_on_a_bad_caldav_section(instance, monkeypatch):
write_settings(config.ROOT, monkeypatch, tasks={"provider": "caldav", **_THRESHOLDS})
checks = doctor.run_doctor()
assert _status(checks, "tasks-provider") == "FAIL"
def test_tasks_provider_ok_but_names_caldav_unreachable_when_configured(instance):
(config.ROOT / config.TASKS_CONFIG_FILENAME).write_text(
json.dumps({
"schema": 1, "provider": "caldav",
"thresholds": {
"stalled_waiting_days": 14, "unpaged_project_weeks": 3,
"someday_stale_months": 5,
},
"caldav": {
"url": "http://127.0.0.1:1/cal/", "username": "u", "app_password": "p",
"inbox_list": "Inbox", "someday_list": "Someday",
},
}),
encoding="utf-8",
)
def test_tasks_provider_ok_but_names_caldav_unreachable_when_configured(instance, monkeypatch):
write_settings(config.ROOT, monkeypatch, tasks={"provider": "caldav", **_THRESHOLDS}, caldav={
"url": "http://127.0.0.1:1/cal/", "username": "u", "app_password": "p",
"inbox_list": "Inbox", "someday_list": "Someday",
})
checks = doctor.run_doctor()
assert _status(checks, "tasks-provider") == "OK"
detail = _detail(checks, "tasks-provider")
@@ -469,6 +435,17 @@ def test_tasks_provider_ok_but_names_caldav_unreachable_when_configured(instance
assert "unreachable" in detail
@pytest.mark.parametrize("name", [".wikitool-remotes.json", ".wikitool-telemetry.json",
".wikitool-upload.json", ".wikitool-tasks.json"])
def test_a_leftover_settings_file_of_an_earlier_version_fails(instance, name):
(config.ROOT / name).write_text("{}", encoding="utf-8")
checks = doctor.run_doctor()
failing = [c for c in checks if c.name == "legacy-settings"]
assert [c.status for c in failing] == ["FAIL"]
assert name in failing[0].detail
assert "tools/wikitool config migrate" in (failing[0].fix or "")
def test_missing_generated_file_fails(instance):
config.LOG_FILE.unlink()
checks = doctor.run_doctor()
@@ -521,16 +498,19 @@ def test_telemetry_never_fails_either_way(instance):
assert _status(doctor.run_doctor(), "telemetry") == "OK"
def _telemetry_with(text: "str | None"):
"""`check_telemetry` against a `.wikitool-telemetry.json` holding `text`
(absent for None), plus what the policy itself makes of it."""
def _telemetry_with(section: "str | None", raw: "str | None" = None):
"""`check_telemetry` against a `telemetry` section of `.wikitool-local.json`
holding the JSON text `section` (absent for None; `raw` replaces the whole
file instead), plus what the policy itself makes of it."""
from chemenu.telemetry import policy
path = config.ROOT / config.TELEMETRY_FILENAME
if text is None:
path = config.ROOT / config.LOCAL_SETTINGS_FILENAME
if raw is not None:
path.write_text(raw, encoding="utf-8")
elif section is None:
path.unlink(missing_ok=True)
else:
path.write_text(text, encoding="utf-8")
path.write_text('{"schema": 1, "telemetry": %s}' % section, encoding="utf-8")
policy.reset_cache()
try:
return doctor.check_telemetry(), policy.resolve(config.ROOT)
@@ -538,32 +518,38 @@ def _telemetry_with(text: "str | None"):
policy.reset_cache()
@pytest.mark.parametrize("text", [None, '{"enabled": false}', '{"schema": 1, "enabled": true, '
@pytest.mark.parametrize("text", [None, '{"enabled": false}', '{"enabled": true, '
'"max_session_bytes": 1000, "keep_sessions": 9}'])
def test_telemetry_stays_ok_for_an_absent_or_conforming_file(instance, text):
def test_telemetry_stays_ok_for_an_absent_or_conforming_section(instance, text):
check, _ = _telemetry_with(text)
assert check.status == "OK"
assert check.fix is None
@pytest.mark.parametrize("text,named,policy_reads", [
("{not json", "unreadable", (True, 5 * 1024 * 1024, 250)),
("[true]", "not a JSON object", (True, 5 * 1024 * 1024, 250)),
("[true]", "object", (True, 5 * 1024 * 1024, 250)),
('{"enabled": "no"}', "enabled", (True, 5 * 1024 * 1024, 250)),
('{"max_session_bytes": 0}', "max_session_bytes", (True, 5 * 1024 * 1024, 250)),
('{"keep_sessions": "12"}', "keep_sessions", (True, 5 * 1024 * 1024, 12)),
('{"enabled": false, "keep_session": 12}', "keep_session", (False, 5 * 1024 * 1024, 250)),
])
def test_telemetry_warns_on_what_the_policy_reads_differently(instance, text, named, policy_reads):
"""The policy stays silent and reads the file exactly as before; `doctor`
"""The policy stays silent and reads the section exactly as before; `doctor`
is where the deviation becomes visible, by field."""
check, pol = _telemetry_with(text)
assert check.status == "WARN"
assert named in check.detail
assert config.TELEMETRY_FILENAME in check.fix
assert "telemetry" in check.fix
assert (pol.enabled, pol.max_session_bytes, pol.keep_sessions) == policy_reads
def test_telemetry_warns_on_an_unreadable_settings_file_and_the_policy_stays_on_its_default(instance):
check, pol = _telemetry_with(None, raw="{not json")
assert check.status == "WARN"
assert "unreadable" in check.detail
assert (pol.enabled, pol.max_session_bytes, pol.keep_sessions) == (True, 5 * 1024 * 1024, 250)
def test_telemetry_reports_the_current_session_count_and_byte_total(instance, monkeypatch):
from chemenu.telemetry.writer import TRACE_FILE, trace_root
@@ -612,14 +598,14 @@ def test_tasks_provider_names_an_active_override(instance, monkeypatch):
profile = config.ROOT / "profile.json"
profile.write_text(
json.dumps({
"schema": 1, "provider": "superproductivity",
"thresholds": {"stalled_waiting_days": 14, "unpaged_project_weeks": 3,
"someday_stale_months": 5},
"schema": 1,
"tasks": {"provider": "superproductivity", **_THRESHOLDS},
"superproductivity": {"access": "api", "api_base_url": "http://127.0.0.1:1",
"api_token": "t"},
"api_token": {"ref": "profile_token"}},
}),
encoding="utf-8",
)
monkeypatch.setenv("WIKITOOL_SECRET_PROFILE_TOKEN", "t")
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(profile))
checks = doctor.run_doctor()
assert _status(checks, "tasks-provider") == "OK"
tools/chemenu/tests/test_git_publish.py
+5 -5
View File
@@ -1306,8 +1306,8 @@ def test_a_commit_left_behind_while_offline_goes_out_with_the_next_reachable_pub
def _allowlist(root, *urls):
(root / config.PUBLISH_REMOTES_FILENAME).write_text(
json.dumps({"schema": 1, "allowed_push_urls": list(urls)}), encoding="utf-8"
(root / config.LOCAL_SETTINGS_FILENAME).write_text(
json.dumps({"schema": 1, "remotes": {"allowed_push_urls": list(urls)}}), encoding="utf-8"
)
@@ -1385,14 +1385,14 @@ def test_no_push_skips_the_gate(repo):
def test_unreadable_allowlist_fails_instead_of_falling_open(repo):
"""A broken file must not be read as 'no restriction' - that would turn a
corrupted safeguard into a silently disabled one."""
(repo / config.PUBLISH_REMOTES_FILENAME).write_text("{not json", encoding="utf-8")
(repo / config.LOCAL_SETTINGS_FILENAME).write_text("{not json", encoding="utf-8")
with pytest.raises(typer.Exit):
git_publish.read_allowed_push_urls()
def test_allowlist_without_a_usable_list_fails(repo):
(repo / config.PUBLISH_REMOTES_FILENAME).write_text(
json.dumps({"schema": 1, "allowed_push_urls": "not-a-list"}), encoding="utf-8"
(repo / config.LOCAL_SETTINGS_FILENAME).write_text(
json.dumps({"schema": 1, "remotes": {"allowed_push_urls": "not-a-list"}}), encoding="utf-8"
)
with pytest.raises(typer.Exit):
git_publish.read_allowed_push_urls()
tools/chemenu/tests/test_legacy_settings.py
+423
View File
@@ -0,0 +1,423 @@
"""The move out of the per-feature settings files of earlier versions
(`chemenu.legacy_settings`, `wikitool config migrate`), and what a leftover of
them does to a command.
The old files are written here by hand, in the shapes they had. Their readers
no longer exist, so "the same effective value" is held against literals that
state what each old file meant - including the defaults of every field it left
out - and read back through the readers the commands use today.
"""
from __future__ import annotations
import json
import subprocess
from dataclasses import asdict
from pathlib import Path
import pytest
from typer.testing import CliRunner
from chemenu import config, config_shapes, legacy_settings, settings, upload
from chemenu.cli import app
from chemenu.tasks import caldav, config as tasks_config, superproductivity
from chemenu.telemetry import policy
runner = CliRunner()
TOKEN = "sekrit-token-value-0451"
APP_PASSWORD = "sekrit-app-password-0815"
THRESHOLDS = {"stalled_waiting_days": 14, "unpaged_project_weeks": 3, "someday_stale_months": 5}
# name -> (old files by file name, what is in effect afterwards)
CASES = {
"telemetry-full": (
{".wikitool-telemetry.json": {"schema": 1, "enabled": False, "max_session_bytes": 1000,
"keep_sessions": 9}},
{"telemetry": {"enabled": False, "max_session_bytes": 1000, "keep_sessions": 9}},
),
"telemetry-defaults": (
{".wikitool-telemetry.json": {"keep_sessions": 12}},
{"telemetry": {"enabled": True, "max_session_bytes": 5 * 1024 * 1024, "keep_sessions": 12}},
),
"telemetry-coerced": (
{".wikitool-telemetry.json": {"enabled": 0, "keep_sessions": "7"}},
{"telemetry": {"enabled": False, "max_session_bytes": 5 * 1024 * 1024, "keep_sessions": 7}},
),
"remotes": (
{".wikitool-remotes.json": {"schema": 1, "allowed_push_urls": ["ssh://git@example.net/a.git",
"ssh://git@example.net/b.git"]}},
{"remotes": ["ssh://git@example.net/a.git", "ssh://git@example.net/b.git"]},
),
"upload-full": (
{".wikitool-upload.json": {
"schema": 1, "identity_header": "X-Forwarded-User", "max_bytes": 2048,
"allowed_extensions": [".md", ".txt"],
"quota": {"submissions_per_day": 4, "bytes_per_day": 9000}}},
{"upload": {"identity_header": "X-Forwarded-User", "max_bytes": 2048,
"allowed_extensions": (".md", ".txt"), "submissions_per_day": 4,
"bytes_per_day": 9000}},
),
"upload-defaults": (
{".wikitool-upload.json": {"max_bytes": 100, "allowed_extensions": [".md"],
"quota": {"submissions_per_day": 1, "bytes_per_day": 5}}},
{"upload": "identity-header-by-default"},
),
"superproductivity-api": (
{".wikitool-tasks.json": {
"schema": 1, "provider": "superproductivity", "thresholds": THRESHOLDS,
"superproductivity": {"access": "api", "api_base_url": "http://127.0.0.1:3876",
"api_token": TOKEN}}},
{"tasks": {"provider": "superproductivity", "thresholds": THRESHOLDS,
"provider_values": {"access": "api", "api_base_url": "http://127.0.0.1:3876",
"api_token": TOKEN, "backups_dir": None}}},
),
"superproductivity-snapshot": (
{".wikitool-tasks.json": {
"schema": 1, "provider": "superproductivity", "thresholds": THRESHOLDS,
"superproductivity": {"access": "snapshot", "backups_dir": "/var/backups/sp"}}},
{"tasks": {"provider": "superproductivity", "thresholds": THRESHOLDS,
"provider_values": {"access": "snapshot", "api_base_url": "http://127.0.0.1:3876",
"api_token": None, "backups_dir": "/var/backups/sp"}}},
),
"caldav": (
{".wikitool-tasks.json": {
"schema": 1, "provider": "caldav", "thresholds": THRESHOLDS,
"caldav": {"url": "https://dav.example.net/u/cal", "username": "anna",
"app_password": APP_PASSWORD, "inbox_list": "Eingang",
"someday_list": "Irgendwann", "exclude_lists": ["Privat", "Archiv"]}}},
{"tasks": {"provider": "caldav", "thresholds": THRESHOLDS,
"provider_values": {"url": "https://dav.example.net/u/cal/", "username": "anna",
"app_password": APP_PASSWORD, "inbox_list": "Eingang",
"someday_list": "Irgendwann",
"exclude_lists": ("Privat", "Archiv")}}},
),
"everything-at-once": (
{
".wikitool-telemetry.json": {"enabled": False},
".wikitool-remotes.json": {"allowed_push_urls": ["ssh://git@example.net/a.git"]},
".wikitool-upload.json": {"max_bytes": 100, "allowed_extensions": [".md"],
"quota": {"submissions_per_day": 1, "bytes_per_day": 5}},
".wikitool-tasks.json": {
"provider": "caldav", "thresholds": THRESHOLDS,
"caldav": {"url": "https://dav.example.net/u/", "username": "anna",
"app_password": APP_PASSWORD, "inbox_list": "Inbox",
"someday_list": "Someday"}},
},
{
"telemetry": {"enabled": False, "max_session_bytes": 5 * 1024 * 1024, "keep_sessions": 250},
"remotes": ["ssh://git@example.net/a.git"],
"upload": "identity-header-by-default",
"tasks": {"provider": "caldav", "thresholds": THRESHOLDS,
"provider_values": {"url": "https://dav.example.net/u/", "username": "anna",
"app_password": APP_PASSWORD, "inbox_list": "Inbox",
"someday_list": "Someday", "exclude_lists": ()}},
},
),
}
def _default(section: str, name: str):
return settings.sections()[section].field(name).default
@pytest.fixture
def checkout(tmp_path, monkeypatch):
"""A git checkout under `tmp_path/work`, with the secrets file kept outside it."""
root = tmp_path / "work"
root.mkdir()
monkeypatch.setattr(config, "ROOT", root)
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg"))
subprocess.run(["git", "init", "-b", "main"], cwd=root, check=True, capture_output=True)
subprocess.run(["git", "config", "user.email", "a@example.net"], cwd=root, check=True)
subprocess.run(["git", "config", "user.name", "A"], cwd=root, check=True)
policy.reset_cache()
yield root
policy.reset_cache()
def _write_old(root: Path, files: dict) -> dict[str, bytes]:
for name, data in files.items():
(root / name).write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
return {name: (root / name).read_bytes() for name in files}
def _snapshot(root: Path, extra: Path | None = None) -> dict[str, bytes]:
found = {str(p.relative_to(root)): p.read_bytes() for p in root.rglob("*")
if p.is_file() and ".git" not in p.relative_to(root).parts}
if extra is not None and extra.exists():
found.update({f"<xdg>/{p.relative_to(extra)}": p.read_bytes() for p in extra.rglob("*") if p.is_file()})
return found
def _effective(root: Path) -> dict:
"""What is in effect for `root`, read through the readers the commands use."""
policy.reset_cache()
pol = policy.resolve(root)
out: dict = {"telemetry": {"enabled": pol.enabled, "max_session_bytes": pol.max_session_bytes,
"keep_sessions": pol.keep_sessions}}
remotes = settings.load(root, only=("remotes",)).section("remotes")
out["remotes"] = list(remotes["allowed_push_urls"]) if remotes.configured else None
cfg = upload.read_config(root)
out["upload"] = None if cfg is None else asdict(cfg)
tasks = tasks_config.read_config(root)
if tasks is None:
out["tasks"] = None
else:
section = tasks.provider_config
if tasks.provider == "caldav":
values = asdict(caldav.CalDAVConfig.from_section(section))
else:
values = asdict(superproductivity.SuperProductivityConfig.from_section(section))
values["backups_dir"] = None if values["backups_dir"] is None else str(values["backups_dir"])
out["tasks"] = {"provider": tasks.provider, "thresholds": asdict(tasks.thresholds),
"provider_values": values}
return out
def _expected(case: dict) -> dict:
"""The `_effective` shape for what a case says is in effect; a feature the
case does not mention is at its defaults."""
out: dict = {
"telemetry": {"enabled": True, "max_session_bytes": 5 * 1024 * 1024, "keep_sessions": 250},
"remotes": None, "upload": None, "tasks": None,
}
for key, value in case.items():
if key == "upload" and isinstance(value, str):
value = {
"identity_header": _default("upload", "identity_header"),
"max_bytes": 100, "allowed_extensions": (".md",),
"submissions_per_day": 1, "bytes_per_day": 5,
}
out[key] = value
return out
def _normal(effective: dict) -> dict:
"""Tuples and lists compare equal."""
return json.loads(json.dumps(effective))
# --- a: the same effective value after the move -------------------------------
@pytest.mark.parametrize("case", CASES)
def test_every_shape_keeps_its_effective_value(checkout, case):
files, expected = CASES[case]
_write_old(checkout, files)
result = legacy_settings.migrate(checkout)
assert sorted(result.removed) == sorted(files)
assert _normal(_effective(checkout)) == _normal(_expected(expected))
# --- b: only the tools file remains; no secret in the tree or the index -------
def test_after_the_move_no_old_file_is_left_and_no_secret_is_in_the_tree_or_the_index(checkout, tmp_path):
files, _ = CASES["everything-at-once"]
files = {**files, ".wikitool-tasks.json": CASES["superproductivity-api"][0][".wikitool-tasks.json"]}
_write_old(checkout, files)
(checkout / ".wikitool-tools.json").write_text('{"schema": 1}', encoding="utf-8")
legacy_settings.migrate(checkout)
assert legacy_settings.present(checkout) == []
assert (checkout / ".wikitool-tools.json").exists()
subprocess.run(["git", "add", "-A", "-f"], cwd=checkout, check=True, capture_output=True)
for secret in (TOKEN, APP_PASSWORD):
grep = subprocess.run(["git", "grep", "--cached", "-n", secret], cwd=checkout,
capture_output=True, text=True)
assert grep.returncode == 1, grep.stdout
for path in checkout.rglob("*"):
if path.is_file() and ".git" not in path.relative_to(checkout).parts:
assert secret not in path.read_text(encoding="utf-8", errors="ignore"), path
secrets_files = list((tmp_path / "xdg").rglob("secrets.json"))
assert len(secrets_files) == 1
assert TOKEN in secrets_files[0].read_text(encoding="utf-8")
def test_the_output_names_settings_and_never_a_secret_value(checkout):
_write_old(checkout, CASES["superproductivity-api"][0])
result = runner.invoke(app, ["config", "migrate"])
assert result.exit_code == 0, result.output
assert TOKEN not in result.output
assert "superproductivity.api_token" in result.output
# --- c: an abort leaves the old files byte-identical --------------------------
@pytest.mark.parametrize("fail_on_write", [2, 3])
def test_a_fault_after_the_first_write_leaves_everything_as_it_was(checkout, tmp_path, monkeypatch, fail_on_write):
files = {
".wikitool-remotes.json": {"allowed_push_urls": ["ssh://git@example.net/a.git"]},
".wikitool-tasks.json": CASES["superproductivity-api"][0][".wikitool-tasks.json"],
}
_write_old(checkout, files)
(checkout / "wikitool.json").write_text(
json.dumps({"schema": 1, "instance_id": "inst-1"}) + "\n", encoding="utf-8")
before = _snapshot(checkout, tmp_path / "xdg")
real, calls = legacy_settings._write, []
def broken(path, data):
calls.append(path)
if len(calls) == fail_on_write:
raise OSError("disk full")
real(path, data)
monkeypatch.setattr(legacy_settings, "_write", broken)
with pytest.raises(legacy_settings.ValidationError) as caught:
legacy_settings.migrate(checkout)
assert "Nothing was changed" in str(caught.value)
assert len(calls) == fail_on_write
assert _snapshot(checkout, tmp_path / "xdg") == before
def test_a_broken_old_file_changes_nothing_and_names_the_file(checkout, tmp_path):
_write_old(checkout, {".wikitool-remotes.json": {"allowed_push_urls": ["ssh://git@example.net/a.git"]}})
(checkout / ".wikitool-upload.json").write_text('{"max_bytes": "many"}', encoding="utf-8")
before = _snapshot(checkout, tmp_path / "xdg")
result = runner.invoke(app, ["config", "migrate"])
assert result.exit_code == 1
assert ".wikitool-upload.json" in result.output
assert _snapshot(checkout, tmp_path / "xdg") == before
def test_a_value_that_collides_with_the_new_settings_changes_nothing(checkout, tmp_path):
_write_old(checkout, {".wikitool-remotes.json": {"allowed_push_urls": ["ssh://git@example.net/a.git"]}})
(checkout / ".wikitool-local.json").write_text(
json.dumps({"schema": 1, "remotes": {"allowed_push_urls": ["ssh://git@example.net/other.git"]}}),
encoding="utf-8")
before = _snapshot(checkout, tmp_path / "xdg")
with pytest.raises(legacy_settings.ValidationError, match="already holds a different value"):
legacy_settings.migrate(checkout)
assert _snapshot(checkout, tmp_path / "xdg") == before
# --- d: a second run changes nothing ------------------------------------------
def test_a_second_run_exits_0_and_changes_no_file(checkout, tmp_path):
_write_old(checkout, CASES["everything-at-once"][0])
first = runner.invoke(app, ["config", "migrate"])
assert first.exit_code == 0, first.output
after_first = _snapshot(checkout, tmp_path / "xdg")
second = runner.invoke(app, ["config", "migrate"])
assert second.exit_code == 0, second.output
assert "Nothing to migrate" in second.output
assert _snapshot(checkout, tmp_path / "xdg") == after_first
def test_a_migrated_setting_that_is_already_there_with_the_same_value_is_not_a_collision(checkout):
_write_old(checkout, {".wikitool-remotes.json": {"allowed_push_urls": ["ssh://git@example.net/a.git"]}})
(checkout / ".wikitool-local.json").write_text(
json.dumps({"schema": 1, "remotes": {"allowed_push_urls": ["ssh://git@example.net/a.git"]}}),
encoding="utf-8")
result = legacy_settings.migrate(checkout)
assert result.removed == [".wikitool-remotes.json"]
# --- e: a leftover is exit 42, FAIL in doctor, silent in the hook -------------
@pytest.mark.parametrize("args,name", [
(["task", "list", "--project", "x"], ".wikitool-tasks.json"),
(["task", "new", "--title", "x", "--inbox"], ".wikitool-tasks.json"),
(["review"], ".wikitool-tasks.json"),
(["publish", "--message", "x"], ".wikitool-remotes.json"),
])
def test_a_leftover_makes_the_command_that_needs_the_setting_exit_42(checkout, monkeypatch, args, name):
(checkout / "kb").mkdir()
monkeypatch.setattr(config, "KB_DIR", checkout / "kb")
(checkout / name).write_text("{}", encoding="utf-8")
result = runner.invoke(app, args)
assert result.exit_code == 42, result.output
assert "tools/wikitool config migrate" in " ".join(result.output.split())
def test_a_tracker_profile_makes_the_old_tasks_file_beside_it_irrelevant(checkout, monkeypatch):
(checkout / ".wikitool-tasks.json").write_text("{}", encoding="utf-8")
profile = checkout / "profile.json"
profile.write_text(json.dumps({"schema": 1, "tasks": {"provider": "caldav", **THRESHOLDS},
"caldav": {"url": "http://127.0.0.1:1/", "username": "u",
"app_password": {"ref": "profile_pw"},
"inbox_list": "I", "someday_list": "S"}}), encoding="utf-8")
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(profile))
monkeypatch.setenv("WIKITOOL_SECRET_PROFILE_PW", "pw")
assert tasks_config.read_config(checkout).provider == "caldav"
def test_the_mcp_server_refuses_to_start_on_an_old_upload_file(checkout, capsys):
pytest.importorskip("mcp")
from chemenu.mcp.__main__ import main
(checkout / ".wikitool-upload.json").write_text('{"max_bytes": 100}', encoding="utf-8")
assert main(["--root", str(checkout)]) == 1
assert "tools/wikitool config migrate" in capsys.readouterr().err
def test_doctor_reports_each_old_file_as_a_fail(checkout):
from chemenu.commands import doctor
_write_old(checkout, {name: {} for name in legacy_settings.FILES.values()})
checks = [c for c in doctor.check_legacy_settings()]
assert [c.status for c in checks] == ["FAIL"] * 4
assert all("tools/wikitool config migrate" in (c.fix or "") for c in checks)
def test_the_telemetry_hook_neither_reads_nor_refuses_over_an_old_file(checkout):
(checkout / ".wikitool-telemetry.json").write_text('{"enabled": false}', encoding="utf-8")
assert policy.resolve(checkout).enabled is True
# --- g: `config set` and the gate flags ---------------------------------------
@pytest.mark.parametrize("key,value", [
("upload.max_bytes", "100"),
("remotes.allowed_push_urls", '["ssh://git@example.net/a.git"]'),
])
def test_config_set_on_a_gated_section_exits_42_and_writes_nothing(checkout, key, value):
result = runner.invoke(app, ["config", "set", key, value])
assert result.exit_code == 42, result.output
assert not (checkout / config.LOCAL_SETTINGS_FILENAME).exists()
def test_config_set_on_telemetry_writes_the_local_file(checkout):
result = runner.invoke(app, ["config", "set", "telemetry.keep_sessions", "7"])
assert result.exit_code == 0, result.output
data = json.loads((checkout / config.LOCAL_SETTINGS_FILENAME).read_text(encoding="utf-8"))
assert data["telemetry"] == {"keep_sessions": 7}
# --- h: the access-path rules of Super Productivity ---------------------------
@pytest.mark.parametrize("section,message", [
({"access": "api"}, "api_token"),
({"access": "api", "api_token": "t", "backups_dir": "/x"}, "backups_dir"),
({"access": "snapshot", "backups_dir": "/x", "api_token": "t"}, "api_token"),
({"access": "snapshot"}, "backups_dir"),
])
def test_a_superproductivity_section_with_the_wrong_fields_for_its_access_fails(checkout, section, message):
_write_old(checkout, {".wikitool-tasks.json": {
"provider": "superproductivity", "thresholds": THRESHOLDS, "superproductivity": section}})
with pytest.raises(legacy_settings.ValidationError, match=message):
legacy_settings.migrate(checkout)
assert (checkout / ".wikitool-tasks.json").exists()
# ... and exactly so when it is read from the new settings
from chemenu.tests.conftest import write_settings
(checkout / ".wikitool-tasks.json").unlink()
mp = pytest.MonkeyPatch()
try:
write_settings(checkout, mp, tasks={"provider": "superproductivity", **THRESHOLDS},
superproductivity=section)
with pytest.raises(legacy_settings.ChemenuError, match=message):
superproductivity.SuperProductivityConfig.from_section(
tasks_config.read_config(checkout).provider_config)
finally:
mp.undo()
tools/chemenu/tests/test_mcp_server.py
+10 -7
View File
@@ -9,7 +9,7 @@ The acceptance criteria that are not about a value coming back:
importable from the server - structural, not filtered;
- every response carries the commit it was computed from;
- telemetry cannot land inside the served checkout;
- the sixth tool, `submit`, exists only when `.wikitool-upload.json` opts a
- the sixth tool, `submit`, exists only when the `upload` section of `.wikitool-local.json` opts a
checkout in, writes only under `mcp-upload/`, and never resolves an
identity from anywhere but the request's own headers.
"""
@@ -44,10 +44,13 @@ from chemenu.mcp.server import ( # noqa: E402 - after the skip guard
_UPLOAD_CFG = {
"schema": 1,
"identity_header": "X-Forwarded-User",
"max_bytes": 1024,
"allowed_extensions": [".md"],
"quota": {"submissions_per_day": 10, "bytes_per_day": 100_000},
"upload": {
"identity_header": "X-Forwarded-User",
"max_bytes": 1024,
"allowed_extensions": [".md"],
"quota_submissions_per_day": 10,
"quota_bytes_per_day": 100_000,
},
}
@@ -348,7 +351,7 @@ def test_a_refused_trace_destination_stops_the_process_with_a_message(monkeypatc
# --- submit (Gitea #32) --------------------------------------------------------
def _arm_upload(corpus: Path) -> None:
(corpus / ".wikitool-upload.json").write_text(json.dumps(_UPLOAD_CFG), encoding="utf-8")
(corpus / ".wikitool-local.json").write_text(json.dumps(_UPLOAD_CFG), encoding="utf-8")
(corpus / "incoming").mkdir(exist_ok=True)
@@ -366,7 +369,7 @@ def test_submit_is_present_once_armed(corpus):
def test_a_malformed_upload_config_refuses_to_build(corpus):
(corpus / ".wikitool-upload.json").write_text("{not json", encoding="utf-8")
(corpus / ".wikitool-local.json").write_text("{not json", encoding="utf-8")
with pytest.raises(ValidationError):
build_server(corpus, check_trace=False)
tools/chemenu/tests/test_new_page.py
+31 -23
View File
@@ -23,32 +23,40 @@ _TASKS_THRESHOLDS = {
_API_TOKEN = "test-token"
@pytest.fixture(autouse=True)
def _tracker_token(monkeypatch):
monkeypatch.setenv("WIKITOOL_SECRET_SP_TEST_TOKEN", _API_TOKEN)
def _write_tracker_settings(root, superproductivity: dict) -> None:
"""The tracker's settings in the tiered files: the instance sections in
`wikitool.json`, the connection in `.wikitool-local.json`. Duplicated
locally rather than imported since test files in this suite do not import
each other (see `instructions/dev/testing-conventions.md`)."""
api = superproductivity.get("access") == "api"
(root / "wikitool.json").write_text(json.dumps({
"schema": 1, "instance_id": "testinstance",
"tasks": {"provider": "superproductivity", **_TASKS_THRESHOLDS},
"superproductivity": {"access": superproductivity["access"]},
}), encoding="utf-8")
local = {k: v for k, v in superproductivity.items() if k != "access"}
if api:
local["api_token"] = {"ref": "sp_test_token"}
(root / ".wikitool-local.json").write_text(
json.dumps({"schema": 1, "superproductivity": local}), encoding="utf-8")
def _write_tasks_config(root, base_url: str) -> None:
"""A minimal `.wikitool-tasks.json` pointing the superproductivity
provider's `access: "api"` path at `base_url` (Gitea #133) - `new
project` only ever has a write path on that access mode. Duplicated
locally rather than imported since test files in this suite do not
import each other (see `instructions/dev/testing-conventions.md` for the
isolation this mirrors one layer up)."""
(root / ".wikitool-tasks.json").write_text(
json.dumps({
"schema": 1, "provider": "superproductivity", "thresholds": _TASKS_THRESHOLDS,
"superproductivity": {"access": "api", "api_base_url": base_url, "api_token": _API_TOKEN},
}),
encoding="utf-8",
)
"""A minimal tracker setup pointing the superproductivity provider's
`access: "api"` path at `base_url` - `new project` only ever has a write
path on that access mode."""
_write_tracker_settings(root, {"access": "api", "api_base_url": base_url})
def _write_tasks_config_snapshot(root, backups_dir) -> None:
"""The other access path (Gitea #133) - used only by the tests below
that confirm `new project` refuses entirely against it."""
(root / ".wikitool-tasks.json").write_text(
json.dumps({
"schema": 1, "provider": "superproductivity", "thresholds": _TASKS_THRESHOLDS,
"superproductivity": {"access": "snapshot", "backups_dir": str(backups_dir)},
}),
encoding="utf-8",
)
"""The other access path - used only by the tests below that confirm
`new project` refuses entirely against it."""
_write_tracker_settings(root, {"access": "snapshot", "backups_dir": str(backups_dir)})
def _write_sp_backup(root, project_titles: list[str]) -> Path:
@@ -252,7 +260,7 @@ def test_new_project_refuses_a_responsibility_outside_the_enum(monkeypatch, kb_d
def test_new_project_says_explicitly_when_no_tracker_is_configured(monkeypatch, kb_dir):
"""No `.wikitool-tasks.json` at all - the page-only state is legitimate
"""No tracker settings at all - the page-only state is legitimate
(#126's own AC) but must be said, not left implicit."""
result = _invoke_new(monkeypatch, kb_dir, [
"new", "project", "--name", "Ohne Tracker", "--set", "responsibility=haus",
tools/chemenu/tests/test_review.py
+18 -20
View File
@@ -21,7 +21,7 @@ from chemenu.cli import app
from chemenu.commands.run_budget import is_exempt
from chemenu.errors import ValidationError
from chemenu.frontmatter_io import write_page
from chemenu.tests.conftest import use_shipped_type_specs
from chemenu.tests.conftest import use_shipped_type_specs, write_settings
from chemenu.review import (
CHECK_NO_OPEN_LOOP,
CHECK_PROJECT_AGE_UNKNOWN,
@@ -79,29 +79,27 @@ def _write_snapshot(root: Path, projects: dict, tasks: dict, tags: dict) -> Path
def _write_tasks_config(root: Path, backups_dir: Path, thresholds: dict | None = None) -> None:
(root / ".wikitool-tasks.json").write_text(
json.dumps({
"schema": 1,
"provider": "superproductivity",
"thresholds": thresholds or THRESHOLDS,
"superproductivity": {"access": "snapshot", "backups_dir": str(backups_dir)},
}),
encoding="utf-8",
write_settings(
root, None,
tasks={"provider": "superproductivity", **(thresholds or THRESHOLDS)},
superproductivity={"access": "snapshot", "backups_dir": str(backups_dir)},
)
@pytest.fixture(autouse=True)
def _caldav_password(monkeypatch):
monkeypatch.setenv("WIKITOOL_SECRET_REVIEW_CALDAV_PASSWORD", _CALDAV_TOKEN[1])
def _write_caldav_tasks_config(root: Path, *, url: str, thresholds: dict | None = None) -> None:
(root / ".wikitool-tasks.json").write_text(
json.dumps({
"schema": 1,
"provider": "caldav",
"thresholds": thresholds or THRESHOLDS,
"caldav": {
"url": url, "username": _CALDAV_TOKEN[0], "app_password": _CALDAV_TOKEN[1],
"inbox_list": "Inbox", "someday_list": "Someday",
},
}),
encoding="utf-8",
write_settings(
root, None,
tasks={"provider": "caldav", **(thresholds or THRESHOLDS)},
caldav={
"url": url, "username": _CALDAV_TOKEN[0],
"app_password": {"ref": "review_caldav_password"},
"inbox_list": "Inbox", "someday_list": "Someday",
},
)
tools/chemenu/tests/test_settings.py
+2 -6
View File
@@ -1,8 +1,8 @@
"""The tiered settings (`chemenu.settings`), `wikitool config set`, and what
`doctor` and `docs verify` report about them.
No real section ships yet, so every test stands in `TRACKER`, a section with one
field of each kind: an instance value, a private checkout value, a secret, a
The tests stand in `TRACKER`, a section of their own with one field of each kind
(the real sections are tested where their feature is): an instance value, a private checkout value, a secret, a
field that names a program, and a plain checkout value. 1Password is a fake `op`
script written into the test's own directory - never a real account; the one
exception is the CI smoke test at the bottom, which only runs where CI has
@@ -57,7 +57,6 @@ TRACKER = Shape(
Field("label", "string", "Plain checkout value.", tier=CHECKOUT, json_type="string"),
),
examples=({},),
unknown_keys=config_shapes.REJECT,
)
@@ -167,9 +166,6 @@ def test_an_unknown_tier_is_refused():
def test_every_shipped_settings_section_is_one():
for shape in config_shapes.SETTINGS_SECTIONS:
assert settings.section_problems(shape) == []
# The four per-feature files predate the tiers and declare none.
for shape in config_shapes.SHAPES:
assert all(not spec.tier for spec in shape.fields)
# --- reading: the right file ----------------------------------------------------
tools/chemenu/tests/test_sp_recorded.py
+7 -5
View File
@@ -85,8 +85,8 @@ def test_the_manifest_names_the_version_and_the_day():
def test_the_api_reader_reads_the_apps_own_answers():
with _replay_server() as base_url:
cfg = sp.SuperProductivityConfig.from_dict(
{"access": "api", "api_base_url": base_url, "api_token": "irrelevant"}
cfg = sp.SuperProductivityConfig(
access=sp.ACCESS_API, backups_dir=None, api_base_url=base_url, api_token="irrelevant"
)
reader = sp.SuperProductivityApiReader(cfg)
@@ -106,8 +106,8 @@ def test_the_api_reader_reads_the_apps_own_answers():
def test_the_inbox_item_is_invisible_to_every_project():
with _replay_server() as base_url:
cfg = sp.SuperProductivityConfig.from_dict(
{"access": "api", "api_base_url": base_url, "api_token": "irrelevant"}
cfg = sp.SuperProductivityConfig(
access=sp.ACCESS_API, backups_dir=None, api_base_url=base_url, api_token="irrelevant"
)
reader = sp.SuperProductivityApiReader(cfg)
seen = {i.title for p in reader.projects() for i in reader.open_items(p.name).items}
@@ -118,7 +118,9 @@ def test_the_snapshot_reader_reads_a_backup_the_app_wrote(tmp_path):
backups = tmp_path / "backups"
backups.mkdir()
shutil.copyfile(FIXTURES / "seed-backup.json", backups / "2026-01-01_000000.json")
cfg = sp.SuperProductivityConfig.from_dict({"access": "snapshot", "backups_dir": str(backups)})
cfg = sp.SuperProductivityConfig(
access=sp.ACCESS_SNAPSHOT, backups_dir=backups, api_base_url=sp.DEFAULT_API_BASE_URL, api_token=None
)
reader = sp.SuperProductivityReader(cfg)
assert {p.name for p in reader.projects()} == PROJECTS
tools/chemenu/tests/test_superproductivity.py
+15 -9
View File
@@ -129,51 +129,57 @@ def cfg(tmp_path) -> sp.SuperProductivityConfig:
)
def _from_dict(data):
from chemenu import config_shapes, settings
return sp.SuperProductivityConfig.from_section(
settings.Section.from_data(config_shapes.SUPERPRODUCTIVITY, data))
# --- from_dict -----------------------------------------------------------------
def test_from_dict_requires_access():
with pytest.raises(ValidationError, match="access"):
sp.SuperProductivityConfig.from_dict({"backups_dir": "/x"})
_from_dict({"backups_dir": "/x"})
def test_from_dict_rejects_an_unknown_access_value():
with pytest.raises(ValidationError, match="access"):
sp.SuperProductivityConfig.from_dict({"access": "auto", "backups_dir": "/x"})
_from_dict({"access": "auto", "backups_dir": "/x"})
def test_from_dict_api_requires_a_token():
with pytest.raises(ValidationError, match="api_token"):
sp.SuperProductivityConfig.from_dict({"access": "api"})
_from_dict({"access": "api"})
def test_from_dict_api_defaults_the_base_url():
cfg_ = sp.SuperProductivityConfig.from_dict({"access": "api", "api_token": "t"})
cfg_ = _from_dict({"access": "api", "api_token": "t"})
assert cfg_.api_base_url == sp.DEFAULT_API_BASE_URL
assert cfg_.access == sp.ACCESS_API
def test_from_dict_api_rejects_a_snapshot_field():
with pytest.raises(ValidationError, match="backups_dir"):
sp.SuperProductivityConfig.from_dict(
_from_dict(
{"access": "api", "api_token": "t", "backups_dir": "/x"}
)
def test_from_dict_snapshot_requires_backups_dir():
with pytest.raises(ValidationError, match="backups_dir"):
sp.SuperProductivityConfig.from_dict({"access": "snapshot"})
_from_dict({"access": "snapshot"})
def test_from_dict_snapshot_rejects_an_api_field():
with pytest.raises(ValidationError, match="api_token"):
sp.SuperProductivityConfig.from_dict(
_from_dict(
{"access": "snapshot", "backups_dir": "/x", "api_token": "t"}
)
def test_from_dict_snapshot_expands_user_in_backups_dir(monkeypatch):
monkeypatch.setenv("HOME", "/home/fixture")
cfg_ = sp.SuperProductivityConfig.from_dict({"access": "snapshot", "backups_dir": "~/backups"})
cfg_ = _from_dict({"access": "snapshot", "backups_dir": "~/backups"})
assert cfg_.backups_dir == Path("/home/fixture/backups")
@@ -181,7 +187,7 @@ def test_from_dict_rejects_db_path_entirely():
"""`db_path` was #133's own casualty - it must not silently work as an
alias for `backups_dir` under either access mode."""
with pytest.raises(ValidationError):
sp.SuperProductivityConfig.from_dict({"access": "snapshot", "db_path": "/x/db.json"})
_from_dict({"access": "snapshot", "db_path": "/x/db.json"})
# --- latest_snapshot_path / schema drift ---------------------------------------
tools/chemenu/tests/test_task_cmd.py
+31 -22
View File
@@ -1,7 +1,7 @@
"""Tests for `wikitool task new` (Gitea #132) - the CLI adapter over
`chemenu.tasks.protocol.TaskWriter.create_item`. Mirrors the fixture shape
`test_new_page.py` uses for `new project`'s own tracker calls: a stub Super
Productivity local REST API on a loopback socket, `.wikitool-tasks.json`
Productivity local REST API on a loopback socket, the tracker settings
pointed at it, and `config.ROOT` repointed at `tmp_path` via the shared
`kb_dir` fixture. No real Super Productivity instance is ever started
(`instructions/dev/testing-conventions.md`).
@@ -16,8 +16,12 @@ from datetime import date, datetime, timezone
from pathlib import Path
from typing import Any
import pytest
from typer.testing import CliRunner
from chemenu import config
from chemenu.tests.conftest import write_settings
runner = CliRunner()
_TASKS_THRESHOLDS = {
@@ -30,13 +34,17 @@ def _ms(year: int, month: int, day: int) -> int:
return int(datetime(year, month, day, tzinfo=timezone.utc).timestamp() * 1000)
@pytest.fixture(autouse=True)
def _api_token_secret(monkeypatch):
monkeypatch.setenv("WIKITOOL_SECRET_TASKCMD_API_TOKEN", _API_TOKEN)
def _write_tasks_config(root: Path, base_url: str) -> None:
(root / ".wikitool-tasks.json").write_text(
json.dumps({
"schema": 1, "provider": "superproductivity", "thresholds": _TASKS_THRESHOLDS,
"superproductivity": {"access": "api", "api_base_url": base_url, "api_token": _API_TOKEN},
}),
encoding="utf-8",
write_settings(
root, None,
tasks={"provider": "superproductivity", **_TASKS_THRESHOLDS},
superproductivity={"access": "api", "api_base_url": base_url,
"api_token": {"ref": "taskcmd_api_token"}},
)
@@ -292,12 +300,10 @@ def test_snapshot_access_has_no_write_path(monkeypatch, kb_dir, tmp_path):
}),
encoding="utf-8",
)
(root / ".wikitool-tasks.json").write_text(
json.dumps({
"schema": 1, "provider": "superproductivity", "thresholds": _TASKS_THRESHOLDS,
"superproductivity": {"access": "snapshot", "backups_dir": str(backups_dir)},
}),
encoding="utf-8",
write_settings(
root, None,
tasks={"provider": "superproductivity", **_TASKS_THRESHOLDS},
superproductivity={"access": "snapshot", "backups_dir": str(backups_dir)},
)
result = _invoke(monkeypatch, kb_dir, [
"task", "new", "--title", "x", "--project", "Ship Chemenu 7.0",
@@ -377,12 +383,10 @@ def test_close_on_snapshot_access_is_refused_the_same_way_as_task_new(monkeypatc
}),
encoding="utf-8",
)
(root / ".wikitool-tasks.json").write_text(
json.dumps({
"schema": 1, "provider": "superproductivity", "thresholds": _TASKS_THRESHOLDS,
"superproductivity": {"access": "snapshot", "backups_dir": str(backups_dir)},
}),
encoding="utf-8",
write_settings(
root, None,
tasks={"provider": "superproductivity", **_TASKS_THRESHOLDS},
superproductivity={"access": "snapshot", "backups_dir": str(backups_dir)},
)
result = _invoke(monkeypatch, kb_dir, ["task", "close", "--id", "t1"])
assert result.exit_code == 1
@@ -415,10 +419,15 @@ def test_an_override_points_a_command_at_another_tracker_without_touching_the_ro
with _api_server(state) as server:
profile_dir = tmp_path / "profiles"
profile_dir.mkdir()
_write_tasks_config(profile_dir, _base_url(server))
profile = profile_dir / ".wikitool-tasks.json"
profile = profile_dir / "profile.json"
profile.write_text(json.dumps({
"schema": 1,
"tasks": {"provider": "superproductivity", **_TASKS_THRESHOLDS},
"superproductivity": {"access": "api", "api_base_url": _base_url(server),
"api_token": {"ref": "taskcmd_api_token"}},
}), encoding="utf-8")
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(profile))
result = _invoke(monkeypatch, kb_dir, ["task", "list", "--project", "Ship Chemenu 7.0"])
assert result.exit_code == 0, result.output
assert "Ueber das Profil" in result.output
assert not (root / ".wikitool-tasks.json").exists()
assert not (root / config.LOCAL_SETTINGS_FILENAME).exists()
tools/chemenu/tests/test_tasks_config.py
+65 -47
View File
@@ -7,26 +7,25 @@ import json
import pytest
from chemenu import config, legacy_settings
from chemenu.errors import ValidationError
from chemenu.tasks import config as tasks_config
from chemenu.tests.conftest import write_settings
VALID = {
"schema": 1,
"provider": "superproductivity",
"thresholds": {
"stalled_waiting_days": 14,
"unpaged_project_weeks": 3,
"someday_stale_months": 5,
},
"superproductivity": {
"access": "snapshot",
"backups_dir": "/tmp/does-not-need-to-exist-for-parsing",
},
}
TASKS = {"provider": "superproductivity", "stalled_waiting_days": 14,
"unpaged_project_weeks": 3, "someday_stale_months": 5}
SP = {"access": "snapshot", "backups_dir": "/tmp/does-not-need-to-exist-for-parsing"}
VALID = {"tasks": TASKS, "superproductivity": SP}
def _write(root, data):
(root / ".wikitool-tasks.json").write_text(json.dumps(data), encoding="utf-8")
@pytest.fixture
def _write(monkeypatch):
def write(root, **changes):
"""The sections of `VALID` (with `changes` replacing whole sections)
into the files, each field where its tier says."""
sections = {**VALID, **changes}
write_settings(root, monkeypatch, **{k: v for k, v in sections.items() if v is not None})
return write
def test_absent_is_none(tmp_path):
@@ -34,13 +33,13 @@ def test_absent_is_none(tmp_path):
def test_malformed_json_is_an_error_not_unconfigured(tmp_path):
(tmp_path / ".wikitool-tasks.json").write_text("{not json", encoding="utf-8")
(tmp_path / config.INSTANCE_SETTINGS_FILENAME).write_text("{not json", encoding="utf-8")
with pytest.raises(ValidationError):
tasks_config.read_config(tmp_path)
def test_a_valid_config_parses(tmp_path):
_write(tmp_path, VALID)
def test_a_valid_config_parses(tmp_path, _write):
_write(tmp_path)
cfg = tasks_config.read_config(tmp_path)
assert cfg.provider == "superproductivity"
assert cfg.thresholds.stalled_waiting_days == 14
@@ -49,68 +48,69 @@ def test_a_valid_config_parses(tmp_path):
assert cfg.provider_config["backups_dir"] == "/tmp/does-not-need-to-exist-for-parsing"
def test_unknown_provider_is_rejected(tmp_path):
data = json.loads(json.dumps(VALID))
data["provider"] = "asana"
_write(tmp_path, data)
def test_unknown_provider_is_rejected(tmp_path, _write):
_write(tmp_path, tasks=dict(TASKS, provider="asana"))
with pytest.raises(ValidationError):
tasks_config.read_config(tmp_path)
def test_missing_threshold_field_is_rejected(tmp_path):
data = json.loads(json.dumps(VALID))
del data["thresholds"]["someday_stale_months"]
_write(tmp_path, data)
def test_missing_threshold_field_is_rejected(tmp_path, _write):
_write(tmp_path, tasks={k: v for k, v in TASKS.items() if k != "someday_stale_months"})
with pytest.raises(ValidationError):
tasks_config.read_config(tmp_path)
def test_non_positive_threshold_is_rejected(tmp_path):
data = json.loads(json.dumps(VALID))
data["thresholds"]["stalled_waiting_days"] = 0
_write(tmp_path, data)
def test_non_positive_threshold_is_rejected(tmp_path, _write):
_write(tmp_path, tasks=dict(TASKS, stalled_waiting_days=0))
with pytest.raises(ValidationError):
tasks_config.read_config(tmp_path)
def test_missing_provider_section_is_rejected(tmp_path):
data = json.loads(json.dumps(VALID))
del data["superproductivity"]
_write(tmp_path, data)
def test_missing_provider_section_is_rejected(tmp_path, _write):
_write(tmp_path, superproductivity=None)
with pytest.raises(ValidationError):
tasks_config.read_config(tmp_path)
def test_a_leftover_file_of_an_earlier_version_is_refused(tmp_path):
(tmp_path / ".wikitool-tasks.json").write_text("{}", encoding="utf-8")
with pytest.raises(legacy_settings.LegacySettingsLeftover, match="config migrate"):
tasks_config.read_config(tmp_path)
# --- the WIKITOOL_TASKS_CONFIG override (Gitea #156) --------------------------------
def test_override_is_read_instead_of_the_root_file(tmp_path, monkeypatch):
PROFILE = VALID
def _profile(path, data=PROFILE):
path.write_text(json.dumps(data), encoding="utf-8")
return path
def test_override_is_read_instead_of_the_root_files(tmp_path, monkeypatch, _write):
root = tmp_path / "root"
root.mkdir()
_write(root, {**VALID, "provider": "asana"}) # would be rejected if it were read
profile = tmp_path / "profile.json"
profile.write_text(json.dumps(VALID), encoding="utf-8")
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(profile))
_write(root, tasks=dict(TASKS, provider="asana")) # would be rejected if it were read
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(_profile(tmp_path / "profile.json")))
assert tasks_config.read_config(root).provider == "superproductivity"
def test_override_works_with_no_root_file_at_all(tmp_path, monkeypatch):
profile = tmp_path / "profile.json"
profile.write_text(json.dumps(VALID), encoding="utf-8")
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(profile))
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(_profile(tmp_path / "profile.json")))
assert tasks_config.read_config(tmp_path / "elsewhere") is not None
def test_a_relative_override_is_taken_from_the_working_directory(tmp_path, monkeypatch):
(tmp_path / "profile.json").write_text(json.dumps(VALID), encoding="utf-8")
_profile(tmp_path / "profile.json")
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", "profile.json")
assert tasks_config.read_config(tmp_path / "elsewhere") is not None
def test_an_override_naming_a_missing_file_is_an_error_never_no_tracker(tmp_path, monkeypatch):
_write(tmp_path, VALID) # a root file must not be a fallback for a broken override
missing = tmp_path / "gone.json"
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(missing))
def test_an_override_naming_a_missing_file_is_an_error_never_no_tracker(tmp_path, monkeypatch, _write):
_write(tmp_path) # root files must not be a fallback for a broken override
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(tmp_path / "gone.json"))
with pytest.raises(ValidationError, match="gone.json"):
tasks_config.read_config(tmp_path)
@@ -126,3 +126,21 @@ def test_a_broken_override_file_names_its_own_path(tmp_path, monkeypatch):
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(profile))
with pytest.raises(ValidationError, match="profile.json"):
tasks_config.read_config(tmp_path)
def test_a_profile_replaces_exactly_the_tracker_sections_of_both_files(tmp_path, monkeypatch, _write):
_write(tmp_path, tasks=dict(TASKS, stalled_waiting_days=99))
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(_profile(tmp_path / "profile.json")))
assert tasks_config.read_config(tmp_path).thresholds.stalled_waiting_days == 14
def test_a_secret_in_a_profile_is_only_ever_a_reference(tmp_path, monkeypatch):
plain = {"tasks": TASKS, "superproductivity": {"access": "api", "api_token": "plain-token-value"}}
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(_profile(tmp_path / "profile.json", plain)))
with pytest.raises(ValidationError) as caught:
tasks_config.read_config(tmp_path)
assert "plain-token-value" not in str(caught.value)
ref = {"tasks": TASKS, "superproductivity": {"access": "api", "api_token": {"ref": "sp_token"}}}
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(_profile(tmp_path / "ref.json", ref)))
monkeypatch.setenv("WIKITOOL_SECRET_SP_TOKEN", "from-env")
assert tasks_config.read_config(tmp_path).provider_config["api_token"] == "from-env"
tools/chemenu/tests/test_telemetry_policy.py
+20 -3
View File
@@ -26,7 +26,7 @@ def _stamp(root: Path) -> None:
def _config_file(root: Path, **fields) -> None:
import json
(root / ".wikitool-telemetry.json").write_text(json.dumps(fields), encoding="utf-8")
(root / ".wikitool-local.json").write_text(json.dumps({"schema": 1, "telemetry": fields}), encoding="utf-8")
def test_a_dev_checkout_defaults_on(tmp_path):
@@ -47,7 +47,7 @@ def test_the_config_file_overrides_the_form_default(tmp_path):
_config_file(tmp_path, enabled=True)
pol = policy.resolve(tmp_path)
assert pol.enabled is True
assert ".wikitool-telemetry.json" in pol.reason
assert ".wikitool-local.json" in pol.reason
def test_the_env_var_overrides_the_config_file_in_both_directions(tmp_path, monkeypatch):
@@ -92,7 +92,7 @@ def test_a_non_positive_or_unparsable_override_is_ignored(tmp_path, monkeypatch)
def test_a_malformed_config_file_is_treated_as_absent(tmp_path):
(tmp_path / ".wikitool-telemetry.json").write_text("{not json", encoding="utf-8")
(tmp_path / ".wikitool-local.json").write_text("{not json", encoding="utf-8")
pol = policy.resolve(tmp_path)
assert pol.enabled is True # falls through to the dev-checkout form default
@@ -125,3 +125,20 @@ def test_resolve_defaults_to_config_root(tmp_path, monkeypatch):
monkeypatch.setattr(config, "ROOT", tmp_path)
_stamp(tmp_path)
assert policy.resolve().enabled is False
def test_a_leftover_file_of_an_earlier_version_is_ignored_silently(tmp_path):
"""The hook never fails and never prompts: the old file counts for nothing
and the form default applies."""
(tmp_path / ".wikitool-telemetry.json").write_text('{"enabled": false}', encoding="utf-8")
pol = policy.resolve(tmp_path)
assert pol.enabled is True
assert ".wikitool-telemetry.json" not in pol.reason
def test_only_the_telemetry_section_is_read_and_leniently(tmp_path):
(tmp_path / ".wikitool-local.json").write_text(
'{"schema": 1, "upload": "garbage", "telemetry": {"keep_sessions": 7, "enabled": "yes"}}',
encoding="utf-8")
pol = policy.resolve(tmp_path)
assert pol.keep_sessions == 7
tools/chemenu/tests/test_tracker_live.py
+7 -4
View File
@@ -77,7 +77,8 @@ def _provisioner(target: LiveTarget):
reader = live.reader_for(target.config)
from chemenu.tasks import caldav as cd
cfg = cd.CalDAVConfig.from_dict(target.config["caldav"])
cfg = cd.CalDAVConfig.from_section(
settings.Section.from_data(config_shapes.CALDAV, target.config["caldav"]))
writer = cd.CalDAVWriter(cfg, reader)
for name in (live.MARKER_PROJECT, cfg.inbox_list, cfg.someday_list):
if name not in {p.name for p in reader.projects()}:
@@ -97,7 +98,9 @@ def test_the_documented_workflow_runs_against_the_real_tracker(
root = kb_dir.parent
profile = tmp_path / "live-profile.json"
profile.write_text(json.dumps(target.config), encoding="utf-8")
assert "plain" not in profile.read_text(encoding="utf-8")
monkeypatch.setenv("WIKITOOL_TASKS_CONFIG", str(profile))
live.export_secrets(target, monkeypatch.setenv)
with capsys.disabled():
print(f"\n[tracker-live] {target.label} - version {target.version} - run {live.RUN_ID}")
record_property("tracker_kind", target.kind)
@@ -229,7 +232,7 @@ def test_a_profile_with_a_home_relative_path_is_refused(tmp_path, monkeypatch):
monkeypatch.setattr(live, "REPO_ROOT", tmp_path)
(tmp_path / live.PROFILE_DIR).mkdir()
(tmp_path / live.PROFILE_DIR / "mine.json").write_text(json.dumps({
"schema": 1, "provider": "superproductivity", "thresholds": live.THRESHOLDS,
"schema": 1, "tasks": {"provider": "superproductivity", **live.THRESHOLDS},
"superproductivity": {"access": "snapshot", "backups_dir": "~/.config/sp/backups"},
}))
with pytest.raises(LiveAbort, match="absolute"):
@@ -246,7 +249,7 @@ def test_a_profile_is_read_exactly_as_written(tmp_path, monkeypatch):
monkeypatch.setattr(live, "REPO_ROOT", tmp_path)
(tmp_path / live.PROFILE_DIR).mkdir()
(tmp_path / live.PROFILE_DIR / "mine.json").write_text(json.dumps({
"schema": 1, "provider": "superproductivity", "thresholds": live.THRESHOLDS,
"schema": 1, "tasks": {"provider": "superproductivity", **live.THRESHOLDS},
"superproductivity": {"access": "snapshot", "backups_dir": str(tmp_path)},
"live_test_version": "19.1.0",
}))
@@ -270,7 +273,7 @@ def test_caldav_target_needs_both_credentials():
"CHEMENU_LIVE_CALDAV_URL": "http://127.0.0.1:5232/u/",
"CHEMENU_LIVE_CALDAV_USER": "u", "CHEMENU_LIVE_CALDAV_PASSWORD": "p",
})
assert target is not None and target.config["provider"] == "caldav"
assert target is not None and target.config["tasks"]["provider"] == "caldav"
def test_item_titles_carry_the_run_prefix():
tools/chemenu/tests/test_upload.py
+26 -43
View File
@@ -13,7 +13,7 @@ from pathlib import Path
import pytest
from chemenu import upload
from chemenu import config, legacy_settings, upload
from chemenu.errors import ValidationError
CONTENT = b"hello world, this is a test submission\n"
@@ -59,72 +59,56 @@ def test_read_config_absent_is_none(tmp_path):
assert upload.read_config(tmp_path) is None
def _section(tmp_path, **fields):
(tmp_path / config.LOCAL_SETTINGS_FILENAME).write_text(
json.dumps({"schema": 1, "upload": fields}), encoding="utf-8")
GOOD = {"max_bytes": 100, "allowed_extensions": [".md"],
"quota_submissions_per_day": 1, "quota_bytes_per_day": 1}
def test_read_config_malformed_json_is_an_error_not_unrestricted(tmp_path):
(tmp_path / ".wikitool-upload.json").write_text("{not json", encoding="utf-8")
(tmp_path / config.LOCAL_SETTINGS_FILENAME).write_text("{not json", encoding="utf-8")
with pytest.raises(ValidationError):
upload.read_config(tmp_path)
def test_read_config_missing_field_is_an_error(tmp_path):
(tmp_path / ".wikitool-upload.json").write_text(
json.dumps({"schema": 1, "max_bytes": 100}), encoding="utf-8"
)
_section(tmp_path, max_bytes=100)
with pytest.raises(ValidationError):
upload.read_config(tmp_path)
def test_read_config_extension_without_dot_is_rejected(tmp_path):
(tmp_path / ".wikitool-upload.json").write_text(
json.dumps({
"schema": 1, "max_bytes": 100, "allowed_extensions": ["md"],
"quota": {"submissions_per_day": 1, "bytes_per_day": 1},
}),
encoding="utf-8",
)
_section(tmp_path, **dict(GOOD, allowed_extensions=["md"]))
with pytest.raises(ValidationError):
upload.read_config(tmp_path)
def test_read_config_rejects_a_non_object_json(tmp_path):
(tmp_path / ".wikitool-upload.json").write_text("[1, 2, 3]", encoding="utf-8")
(tmp_path / config.LOCAL_SETTINGS_FILENAME).write_text("[1, 2, 3]", encoding="utf-8")
with pytest.raises(ValidationError):
upload.read_config(tmp_path)
@pytest.mark.parametrize("value", [-1, 0])
def test_read_config_rejects_a_non_positive_max_bytes(tmp_path, value):
(tmp_path / ".wikitool-upload.json").write_text(
json.dumps({
"schema": 1, "max_bytes": value, "allowed_extensions": [".md"],
"quota": {"submissions_per_day": 1, "bytes_per_day": 1},
}),
encoding="utf-8",
)
_section(tmp_path, **dict(GOOD, max_bytes=value))
with pytest.raises(ValidationError):
upload.read_config(tmp_path)
@pytest.mark.parametrize("key,value", [("submissions_per_day", 0), ("bytes_per_day", -5)])
@pytest.mark.parametrize("key,value", [("quota_submissions_per_day", 0), ("quota_bytes_per_day", -5)])
def test_read_config_rejects_non_positive_quota_values(tmp_path, key, value):
quota = {"submissions_per_day": 1, "bytes_per_day": 1}
quota[key] = value
(tmp_path / ".wikitool-upload.json").write_text(
json.dumps({"schema": 1, "max_bytes": 100, "allowed_extensions": [".md"], "quota": quota}),
encoding="utf-8",
)
_section(tmp_path, **dict(GOOD, **{key: value}))
with pytest.raises(ValidationError):
upload.read_config(tmp_path)
def test_read_config_valid_file_parses(tmp_path):
(tmp_path / ".wikitool-upload.json").write_text(
json.dumps({
"schema": 1, "identity_header": "X-User", "max_bytes": 2048,
"allowed_extensions": [".MD", ".pdf"],
"quota": {"submissions_per_day": 3, "bytes_per_day": 999},
}),
encoding="utf-8",
)
_section(tmp_path, identity_header="X-User", max_bytes=2048, allowed_extensions=[".MD", ".pdf"],
quota_submissions_per_day=3, quota_bytes_per_day=999)
cfg = upload.read_config(tmp_path)
assert cfg.identity_header == "X-User"
assert cfg.max_bytes == 2048
@@ -134,17 +118,16 @@ def test_read_config_valid_file_parses(tmp_path):
def test_read_config_default_identity_header(tmp_path):
(tmp_path / ".wikitool-upload.json").write_text(
json.dumps({
"schema": 1, "max_bytes": 10,
"allowed_extensions": [".md"],
"quota": {"submissions_per_day": 1, "bytes_per_day": 10},
}),
encoding="utf-8",
)
_section(tmp_path, **GOOD)
assert upload.read_config(tmp_path).identity_header == upload.DEFAULT_IDENTITY_HEADER
def test_read_config_refuses_a_file_of_an_earlier_version(tmp_path):
(tmp_path / ".wikitool-upload.json").write_text("{}", encoding="utf-8")
with pytest.raises(legacy_settings.LegacySettingsLeftover, match="config migrate"):
upload.read_config(tmp_path)
# --- the write choke point ----------------------------------------------------
def test_write_primitive_refuses_a_relative_escape(tmp_path):
tools/chemenu/tests/tracker_live.py
+32 -12
View File
@@ -19,6 +19,11 @@ the environment, none of them a fixed address:
provision the marker calendar in (Radicale in CI).
- `CHEMENU_LIVE_PROFILE`: a profile file `.wikitool-tasks.d/<name>.json` in the
checkout - a tracker of the user's own, used exactly as configured.
A profile is in the format of `.wikitool-local.json`, restricted to the sections `tasks`,
`superproductivity` and `caldav`. A secret in it is only ever a reference; the suite's own
throwaway tokens travel in `LiveTarget.secrets` and reach the run as
`WIKITOOL_SECRET_<NAME>`, never as a file.
"""
from __future__ import annotations
@@ -32,7 +37,7 @@ from datetime import date, timedelta
from pathlib import Path
from typing import Any, Callable, Optional
from chemenu import tasks
from chemenu import config, config_shapes, settings, tasks
from chemenu.tasks import config as tasks_config
from chemenu.tasks.protocol import TaskReader, find_project, normalize_project_name
from chemenu.tests.sp_headless import LiveAbort
@@ -52,6 +57,7 @@ class LiveTarget:
label: str
version: str
config: dict[str, Any]
secrets: dict[str, str] = field(default_factory=dict)
cleanup: Callable[[], None] = field(default=lambda: None)
@@ -110,13 +116,13 @@ def caldav_target_from_env(environ: "dict[str, str] | os._Environ[str]" = os.env
version=environ.get("CHEMENU_LIVE_CALDAV_VERSION", "unknown"),
config={
"schema": 1,
"provider": "caldav",
"thresholds": THRESHOLDS,
"tasks": {"provider": "caldav", **THRESHOLDS},
"caldav": {
"url": url, "username": user, "app_password": password,
"url": url, "username": user, "app_password": {"ref": "live_caldav_app_password"},
"inbox_list": "Inbox", "someday_list": "Someday",
},
},
secrets={"live_caldav_app_password": password},
)
@@ -140,12 +146,13 @@ def sp_target_from_env(environ: "dict[str, str] | os._Environ[str]" = os.environ
version=sp_headless.installed_version(executable),
config={
"schema": 1,
"provider": "superproductivity",
"thresholds": THRESHOLDS,
"tasks": {"provider": "superproductivity", **THRESHOLDS},
"superproductivity": {
"access": "api", "api_base_url": process.base_url, "api_token": process.token,
"access": "api", "api_base_url": process.base_url,
"api_token": {"ref": "live_sp_api_token"},
},
},
secrets={"live_sp_api_token": process.token},
cleanup=cleanup,
)
@@ -164,13 +171,26 @@ def build_target(kind: str, environ: "dict[str, str] | os._Environ[str]" = os.en
# --- guard and scenario ------------------------------------------------------------------
def export_secrets(target: LiveTarget, setenv: Callable[[str, str], None]) -> None:
"""Hand the target's throwaway secrets to the run, the one way a secret reaches it."""
for name, value in target.secrets.items():
setenv(config.ENV_SECRET_PREFIX + name.upper(), value)
def reader_for(target_config: dict[str, Any]) -> TaskReader:
"""The reader `target_config` describes, built the way the commands build it."""
provider = target_config["provider"]
"""The reader `target_config` describes, built the way the commands build it. A
reference in it resolves from the environment, so `export_secrets` comes first."""
tracker = settings.Section.from_data(config_shapes.TASKS, target_config["tasks"])
provider = tracker["provider"]
shape = next(s for s in config_shapes.SETTINGS_SECTIONS if s.key == provider)
cfg = tasks_config.TasksConfig(
provider=provider,
thresholds=tasks_config.Thresholds(**target_config["thresholds"]),
provider_config=target_config[provider],
thresholds=tasks_config.Thresholds(
stalled_waiting_days=tracker["stalled_waiting_days"],
unpaged_project_weeks=tracker["unpaged_project_weeks"],
someday_stale_months=tracker["someday_stale_months"],
),
provider_config=settings.Section.from_data(shape, target_config[provider]),
)
return tasks.build_reader(cfg)
@@ -201,6 +221,6 @@ def overdue_day(today: date) -> date:
__all__ = [
"ITEM_PREFIX", "KINDS", "LiveAbort", "LiveTarget", "MARKER_PROJECT", "THRESHOLDS",
"build_target", "guard", "normalize_project_name", "overdue_day", "reader_for",
"build_target", "export_secrets", "guard", "normalize_project_name", "overdue_day", "reader_for",
"required_kinds", "run_id_titles",
]
tools/chemenu/upload.py
+20 -34
View File
@@ -49,7 +49,7 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Any, Optional
from chemenu import config, config_shapes
from chemenu import config, config_shapes, legacy_settings, settings
from chemenu.errors import ValidationError
DEFAULT_IDENTITY_HEADER = config_shapes.UPLOAD.field("identity_header").default
@@ -65,8 +65,7 @@ _FORBIDDEN_FILENAME_CHARS = ("/", "\\", "\x00")
@dataclass(frozen=True)
class UploadConfig:
"""The opt-in, read from `.wikitool-upload.json` - see
`config.UPLOAD_CONFIG_FILENAME`."""
"""The opt-in, read from the `upload` section of `.wikitool-local.json`."""
identity_header: str
max_bytes: int
@@ -76,43 +75,30 @@ class UploadConfig:
def read_config(root: "Path | str") -> Optional[UploadConfig]:
"""The upload opt-in for `root`, or `None` when it is absent - which means
the write path does not exist, not that it is unrestricted.
"""The upload opt-in for `root`, or `None` when the `upload` section is not
set - which means the write path does not exist, not that it is
unrestricted.
A malformed file is a `ValidationError`, never a silent "no limits": this
file decides whether an unauthenticated write path is offered at all, so a
A malformed section is a `ValidationError`, never a silent "no limits": it
decides whether an unauthenticated write path is offered at all, so a
corrupted safeguard must not read as a disabled one - the same posture
`git_publish.read_allowed_push_urls` takes for `.wikitool-remotes.json`,
`git_publish.read_allowed_push_urls` takes for the `remotes` section,
deliberately not the "ignore what does not parse" posture
`telemetry.policy` takes for its own config, because that one only ever
narrows an existing on/off default and this one creates a capability that
otherwise does not exist.
`telemetry.policy` takes for its own, because that one only ever narrows an
existing on/off default and this one creates a capability that otherwise
does not exist. The file of an earlier version that is still there raises
`LegacySettingsLeftover` rather than reading as "not configured".
"""
path = Path(root) / config.UPLOAD_CONFIG_FILENAME
if not path.is_file():
legacy_settings.refuse_leftover(root, "upload")
section = settings.load(root, only=(config_shapes.UPLOAD.key,)).section(config_shapes.UPLOAD.key)
if not section.configured:
return None
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise ValidationError(
f"{config.UPLOAD_CONFIG_FILENAME} is unreadable ({exc}). It decides whether the "
"upload tool is offered at all, so a broken file is not treated as 'no limits' - "
"fix it or delete it deliberately."
) from exc
shape = config_shapes.UPLOAD
checked = config_shapes.check(shape, data)
if checked.problems:
raise ValidationError(
f"{config.UPLOAD_CONFIG_FILENAME}: {checked.message()}. "
f"Expected: {config_shapes.render_expected(shape)}"
)
values = checked.values
return UploadConfig(
identity_header=values["identity_header"],
max_bytes=values["max_bytes"],
allowed_extensions=values["allowed_extensions"],
submissions_per_day=values["quota"]["submissions_per_day"],
bytes_per_day=values["quota"]["bytes_per_day"],
identity_header=section["identity_header"],
max_bytes=section["max_bytes"],
allowed_extensions=section["allowed_extensions"],
submissions_per_day=section["quota_submissions_per_day"],
bytes_per_day=section["quota_bytes_per_day"],
)