Chemenu 2.1.0 - deterministischer Wissenskompiler
Chemenu kompiliert Rohnotizen zu einem verlinkten, quellengebundenen Wiki: raw/ -> types/ + tools/ -> kb/ -> reports/. Was mechanisch ist, macht tools/wikitool; was Urteil braucht, macht ein Agent unter Contracts, deren Grenzen in Code durchgesetzt sind statt im Prompt. Dieser Commit ist der Startpunkt der oeffentlichen Historie. Die vorherige Entwicklung fand in einer privaten Instanz statt und ist nicht Teil dieses Repositorys; ihre Erzaehlung steht vollstaendig in CHANGES.md, das mit 44 Eintraegen von 0.1.0 bis 2.1.0 erhalten geblieben ist. Der mitgelieferte Korpus ist ein Testbett und eine Demo: 170 Seiten ueber den Stack selbst - Gates, Lint, Versionierung, Suche, das Wiki-Muster. Er dokumentiert das Werkzeug mit den eigenen Mitteln des Werkzeugs. Lizenz: AGPL-3.0 fuer den Stack (tools/, types/), CC-BY-4.0 fuer die Inhalte. Die Grenze zwischen beiden ist der Dateiplan, den dist export berechnet - siehe NOTICE.
This commit is contained in:
@@ -0,0 +1,442 @@
|
||||
"""`wikitool dist export` - build a distributable, contentless copy of this
|
||||
repo's machinery.
|
||||
|
||||
`export` copies the pipeline's schema/compiler/control-plane layers (types/,
|
||||
tools/, instructions/, the stage contracts, every kb/*/COLLECTION.md) into an
|
||||
empty target, with no kb/ pages, no raw/ content, and no git history - see
|
||||
instructions/setup-instance.md for what happens after. It never calls git.
|
||||
|
||||
Three independent exclusion mechanisms feed the plan, for three different
|
||||
shapes of "does not belong in someone else's instance":
|
||||
|
||||
- Every copied text file passes through `strip_markers()`, which removes any
|
||||
region between `<!-- dist:strip-start -->` and `<!-- dist:strip-end -->`,
|
||||
markers included - for dev-only *content inside* a file that is otherwise
|
||||
shipped (e.g. a routing line in AGENTS.md).
|
||||
- `instructions/dev/` is pruned from the copy wholesale - for dev-only
|
||||
*whole files* (procedures and the skill that switches an agent into
|
||||
tool-development mode). One-way: nothing reconstructs it in a distributed
|
||||
instance, on purpose - see instructions/dev/ itself for the current
|
||||
contents and AGENTS.md's routing line for what a dev instance sees instead.
|
||||
- Build output under `tools/` is dropped, by directory (`TOOLS_EXCLUDE_DIRS`)
|
||||
where it has one, and by filename (`_is_coverage_output`) where it does not.
|
||||
Not dev-only but *derived*: recomputable, and measured against this repo's
|
||||
own test run rather than the receiving instance's.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from typing import Callable, NamedTuple, Optional, Union
|
||||
|
||||
import typer
|
||||
|
||||
from chemenu import config, kb_collections, kb_state, version as version_mod
|
||||
from chemenu.commands._util import fail, rel_path, success, today_iso
|
||||
|
||||
app = typer.Typer(help="Build a distributable copy of the wiki machinery.")
|
||||
|
||||
MARKER_START = "<!-- dist:strip-start -->"
|
||||
MARKER_END = "<!-- dist:strip-end -->"
|
||||
|
||||
DIST_TEMPLATES_DIR = Path(__file__).resolve().parent.parent / "dist_templates"
|
||||
|
||||
# Root files copied verbatim (after marker-stripping). INSTALL.md is optional
|
||||
# here: it does not exist until the distribution docs land, and `export`
|
||||
# must not fail just because a later stage of the same repo hasn't shipped
|
||||
# yet.
|
||||
#
|
||||
# The personalization *templates* ship; the filled `USER.md`/`SOUL.md` never
|
||||
# do. This allowlist is what makes that split automatic - a file is copied
|
||||
# because it is named here, so an instance's own personalization is excluded
|
||||
# by construction rather than by a rule someone has to remember.
|
||||
#
|
||||
# `ENVIRONMENT.md.template` rides the same split for the same reason: a
|
||||
# distribution can describe what the file is for, but never what a particular
|
||||
# checkout's harness, MCP servers and remotes are. The filled `ENVIRONMENT.md`
|
||||
# is additionally gitignored, so it is excluded twice over.
|
||||
#
|
||||
# `CLAUDE.md` is harness glue, not a second control plane: Claude Code loads it
|
||||
# and does not load `AGENTS.md`, so it ships for the same reason
|
||||
# `.claude/settings.json` does - a distributed instance running that harness
|
||||
# would otherwise start every session without the control plane.
|
||||
ROOT_FILES = (
|
||||
"AGENTS.md", "CLAUDE.md", "README.md", "EVALS.md", "INSTALL.md", ".gitignore", "VERSION",
|
||||
*config.LICENSE_FILES,
|
||||
*config.PERSONALIZATION_TEMPLATES,
|
||||
config.ENVIRONMENT_TEMPLATE,
|
||||
)
|
||||
|
||||
# The one part of ROOT_FILES that may not be quietly skipped. Every other entry
|
||||
# copies only `if source.is_file()`, which is right for `INSTALL.md` (it did not
|
||||
# exist until the distribution docs landed) and wrong for a licence: an export
|
||||
# that silently omits it hands the receiving instance the AGPL-covered `tools/`
|
||||
# tree with no licence text, which is a violation the moment that instance is
|
||||
# pushed anywhere public. Missing means the export is broken, not minimal.
|
||||
REQUIRED_ROOT_FILES = config.LICENSE_FILES
|
||||
|
||||
# Harness-specific session-tracing config: generic machinery (feeds
|
||||
# tools/chemenu/telemetry/ and tools/trace_ingest.py via EVALS.md), not
|
||||
# personal state - unlike `.obsidian/`/`.vscode/`, which are never copied.
|
||||
HOOK_DIRS = (".github/hooks", ".vibe")
|
||||
|
||||
# tools/ subpaths never copied - build/venv/cache artifacts, not machinery.
|
||||
# `htmlcov/` is coverage.py's HTML report: derived output, and a large tree of
|
||||
# it, measured against the source repo's own test run. `.coveragerc` beside it
|
||||
# *does* ship, the same way `pytest.ini` does - it is configuration, not output.
|
||||
TOOLS_EXCLUDE_DIRS = {".venv", "__pycache__", ".pytest_cache", ".wikitool_session", "htmlcov"}
|
||||
|
||||
# The rest of coverage's output lands beside the code rather than in a directory
|
||||
# of its own - `.coverage`, `coverage.xml`, and `.coverage.<host>.<pid>` under a
|
||||
# parallel run - so a directory exclusion cannot reach it. Same argument as
|
||||
# `reports/`: derived, recomputable, and about the source repo rather than about
|
||||
# the instance that would receive it.
|
||||
COVERAGE_OUTPUT_NAMES = frozenset({".coverage", "coverage.xml"})
|
||||
|
||||
|
||||
def _is_coverage_output(filename: str) -> bool:
|
||||
return filename in COVERAGE_OUTPUT_NAMES or filename.startswith(".coverage.")
|
||||
|
||||
|
||||
# instructions/dev/ holds stack-development-only procedures and the skill
|
||||
# that switches an agent into tool-development mode - never shipped to a
|
||||
# distributed instance. One-way: there is no `enable-dev`-style command that
|
||||
# reconstructs it afterwards, unlike the marker-block content below.
|
||||
INSTRUCTIONS_EXCLUDE_DIRS = {"dev"}
|
||||
|
||||
# Fixed by raw/CONTRACT.md's routing table, unlike kb/'s areas (which are
|
||||
# organic - see kb/CONTRACT.md - so `export` does not manufacture them).
|
||||
RAW_SUBDIRS = ("articles", "documents", "notes", "assets")
|
||||
|
||||
# Stage contracts that are not collections and carry no pages: copied as a
|
||||
# single file each, nothing else from their directory.
|
||||
CONTRACT_ONLY_STAGES = ("raw/CONTRACT.md", "reports/CONTRACT.md", "work/CONTRACT.md")
|
||||
|
||||
# Single tracked files copied out of an otherwise-untouched, partially-ignored
|
||||
# directory. `.claude/` holds the harness's own session-tracing config
|
||||
# (`settings.json`, tracked) alongside generated skill copies and personal
|
||||
# untracked state (`.claude/skills/`, `.claude/settings.local.json`) - neither
|
||||
# of which belongs in a distribution. Adding `.claude` to HOOK_DIRS would copy
|
||||
# the whole directory, skills included; a single-file entry avoids that
|
||||
# without needing an exclude set HOOK_DIRS doesn't otherwise carry.
|
||||
SINGLE_FILES = (".claude/settings.json",)
|
||||
|
||||
Content = Union[str, bytes]
|
||||
|
||||
|
||||
class PlannedFile(NamedTuple):
|
||||
content: Content
|
||||
executable: bool = False
|
||||
|
||||
|
||||
_MARKER_TOKEN_RE = re.compile(re.escape(MARKER_START) + "|" + re.escape(MARKER_END))
|
||||
# The leading/trailing `\n?` consume the blank line on each side of the
|
||||
# block - the convention is that a marker block always sits as its own
|
||||
# paragraph. Without eating both, a strip leaves two blank lines where the
|
||||
# clean file only ever had one.
|
||||
_MARKER_BLOCK_RE = re.compile(
|
||||
r"\n?" + re.escape(MARKER_START) + r".*?" + re.escape(MARKER_END) + r"\n?", re.DOTALL
|
||||
)
|
||||
|
||||
|
||||
def _validate_markers(text: str, label: str) -> None:
|
||||
"""A marker file must be a sequence of well-formed, non-nested
|
||||
start/end pairs. Malformed markers would make `strip_markers` remove
|
||||
either too little or too much, silently - this fails loudly instead."""
|
||||
depth = 0
|
||||
for match in _MARKER_TOKEN_RE.finditer(text):
|
||||
if match.group() == MARKER_START:
|
||||
if depth != 0:
|
||||
fail(f"{label}: nested dist:strip-start markers are not supported")
|
||||
depth = 1
|
||||
else:
|
||||
if depth != 1:
|
||||
fail(f"{label}: dist:strip-end without a matching dist:strip-start")
|
||||
depth = 0
|
||||
if depth != 0:
|
||||
fail(f"{label}: dist:strip-start without a matching dist:strip-end")
|
||||
|
||||
|
||||
def strip_markers(text: str) -> str:
|
||||
"""Remove every marked region, markers included. Generic by design: it
|
||||
does not matter what is inside, or how many regions a file has."""
|
||||
return _MARKER_BLOCK_RE.sub("", text)
|
||||
|
||||
|
||||
def _is_executable(path: Path) -> bool:
|
||||
return bool(path.stat().st_mode & stat.S_IXUSR)
|
||||
|
||||
|
||||
def _read_planned_file(path: Path, label: str) -> PlannedFile:
|
||||
executable = _is_executable(path)
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return PlannedFile(path.read_bytes(), executable)
|
||||
# Marker syntax is an HTML/Markdown comment convention, scoped to .md
|
||||
# files on purpose: applying it to every text file would let the marker
|
||||
# strings themselves - inline here as Python string literals - match as
|
||||
# a region in this file's own source when tools/ gets copied, and eat
|
||||
# the code between them.
|
||||
if path.suffix != ".md":
|
||||
return PlannedFile(text, executable)
|
||||
_validate_markers(text, label)
|
||||
return PlannedFile(strip_markers(text), executable)
|
||||
|
||||
|
||||
def _copy_tree(
|
||||
source_root: Path,
|
||||
dest_prefix: str,
|
||||
exclude_dirs: frozenset[str],
|
||||
exclude_file: Optional[Callable[[str], bool]] = None,
|
||||
) -> dict[str, PlannedFile]:
|
||||
"""Every file under source_root, marker-stripped, keyed by its
|
||||
destination-relative path. Excluded directories are pruned during the
|
||||
walk rather than filtered after, so a large `.venv/` is never read.
|
||||
`exclude_file` drops individual files by name, for output that lands
|
||||
beside the code instead of in a directory a prune could catch."""
|
||||
files: dict[str, PlannedFile] = {}
|
||||
if not source_root.is_dir():
|
||||
return files
|
||||
for dirpath, dirnames, filenames in os.walk(source_root):
|
||||
dirnames[:] = sorted(d for d in dirnames if d not in exclude_dirs)
|
||||
for filename in sorted(filenames):
|
||||
if exclude_file is not None and exclude_file(filename):
|
||||
continue
|
||||
path = Path(dirpath) / filename
|
||||
relative = path.relative_to(source_root).as_posix()
|
||||
dest_rel = f"{dest_prefix}/{relative}"
|
||||
files[dest_rel] = _read_planned_file(path, dest_rel)
|
||||
return files
|
||||
|
||||
|
||||
def _digest(content: Content) -> str:
|
||||
data = content if isinstance(content, bytes) else content.encode("utf-8")
|
||||
return "sha256:" + hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def build_stamp(plan: dict[str, PlannedFile], origin: "Origin") -> str:
|
||||
"""The release stamp written into every export.
|
||||
|
||||
Two jobs. The version and origin fields are what `version check` compares
|
||||
against a release feed - without them an instance cannot tell which stack
|
||||
it is running. The per-file digests are for the update *after* detection:
|
||||
they record what the machinery looked like when it was installed, which is
|
||||
the only way a later upgrade can tell a file the instance edited from one
|
||||
it merely received. Nothing reads them today; writing them now is what
|
||||
keeps that upgrade from needing a format change.
|
||||
"""
|
||||
stamp = {
|
||||
"schema": version_mod.STAMP_SCHEMA,
|
||||
"version": str(version_mod.read_version()),
|
||||
"exported_at": today_iso(),
|
||||
"source_repo": origin.source_repo,
|
||||
"source_commit": origin.source_commit,
|
||||
"release_url": origin.release_url,
|
||||
"update_url": origin.update_url or version_mod.DEFAULT_UPDATE_URL,
|
||||
"files": {relative: _digest(planned.content) for relative, planned in sorted(plan.items())},
|
||||
}
|
||||
return json.dumps(stamp, indent=2, sort_keys=False) + "\n"
|
||||
|
||||
|
||||
class Origin(NamedTuple):
|
||||
"""Where this export came from. Supplied by the caller (the release
|
||||
workflow knows the commit and the release URL); `dist export` itself never
|
||||
calls git, so it cannot discover any of it."""
|
||||
|
||||
source_repo: Optional[str] = None
|
||||
source_commit: Optional[str] = None
|
||||
release_url: Optional[str] = None
|
||||
update_url: Optional[str] = None
|
||||
|
||||
|
||||
def build_plan(origin: Optional[Origin] = None) -> dict[str, PlannedFile]:
|
||||
"""Every (destination-relative path -> planned file) the export writes."""
|
||||
plan: dict[str, PlannedFile] = {}
|
||||
|
||||
missing_licences = [
|
||||
name for name in REQUIRED_ROOT_FILES if not (config.ROOT / name).is_file()
|
||||
]
|
||||
if missing_licences:
|
||||
fail(
|
||||
"export would ship code without its licence: "
|
||||
+ ", ".join(missing_licences)
|
||||
+ " missing from the source tree. Restore them before exporting - a "
|
||||
"distribution carrying tools/ without LICENSE is a copyleft violation "
|
||||
"the moment the receiving instance is published."
|
||||
)
|
||||
|
||||
for name in ROOT_FILES:
|
||||
source = config.ROOT / name
|
||||
if source.is_file():
|
||||
plan[name] = _read_planned_file(source, name)
|
||||
|
||||
plan.update(_copy_tree(config.INSTRUCTIONS_DIR, "instructions", frozenset(INSTRUCTIONS_EXCLUDE_DIRS)))
|
||||
plan.update(_copy_tree(config.TYPES_DIR, "types", frozenset()))
|
||||
plan.update(_copy_tree(
|
||||
config.ROOT / "tools", "tools", frozenset(TOOLS_EXCLUDE_DIRS), _is_coverage_output
|
||||
))
|
||||
for hook_dir in HOOK_DIRS:
|
||||
plan.update(_copy_tree(config.ROOT / hook_dir, hook_dir, frozenset()))
|
||||
|
||||
kb_contract = config.KB_DIR / "CONTRACT.md"
|
||||
if kb_contract.is_file():
|
||||
plan["kb/CONTRACT.md"] = _read_planned_file(kb_contract, "kb/CONTRACT.md")
|
||||
for collection in kb_collections.iter_kb_collections():
|
||||
rel = f"kb/{collection.name}/COLLECTION.md"
|
||||
plan[rel] = _read_planned_file(collection / "COLLECTION.md", rel)
|
||||
|
||||
for relative in CONTRACT_ONLY_STAGES:
|
||||
source = config.ROOT / relative
|
||||
if source.is_file():
|
||||
plan[relative] = _read_planned_file(source, relative)
|
||||
|
||||
for relative in SINGLE_FILES:
|
||||
source = config.ROOT / relative
|
||||
if source.is_file():
|
||||
plan[relative] = _read_planned_file(source, relative)
|
||||
|
||||
for sub in RAW_SUBDIRS:
|
||||
plan[f"raw/{sub}/.gitkeep"] = PlannedFile("")
|
||||
|
||||
plan["kb/log.md"] = PlannedFile((DIST_TEMPLATES_DIR / "log.md").read_text(encoding="utf-8"))
|
||||
plan["CHANGES.md"] = PlannedFile((DIST_TEMPLATES_DIR / "CHANGES.md").read_text(encoding="utf-8"))
|
||||
|
||||
# A fresh instance's content is empty, so it is trivially in the shape this
|
||||
# machinery expects - which is exactly what makes the initial declaration
|
||||
# safe to write here rather than leaving it to `migrate baseline`. Only an
|
||||
# instance predating this file has to answer that question by hand.
|
||||
plan[kb_state.KB_STATE_FILENAME] = PlannedFile(
|
||||
kb_state.render_kb_state(version_mod.read_version(), [])
|
||||
)
|
||||
|
||||
# Last, so it can digest everything above it. It is the one file in the
|
||||
# export that describes the export rather than being copied into it.
|
||||
plan[version_mod.RELEASE_STAMP_FILENAME] = PlannedFile(
|
||||
build_stamp(plan, origin or Origin())
|
||||
)
|
||||
|
||||
return plan
|
||||
|
||||
|
||||
# Content that must never appear in a plan, expressed structurally rather than
|
||||
# by matching text. Three allowlists feed `build_plan`, and each one holds only
|
||||
# because someone remembered the rule when they edited it - nothing re-checks
|
||||
# the result. This does.
|
||||
#
|
||||
# The checks are deliberately structural: a filled personalization file, a kb
|
||||
# page, a raw source, a dev-only instruction. A text-pattern scan (hostnames,
|
||||
# IP literals) was considered and rejected - the project's own host legitimately
|
||||
# appears in INSTALL.md and version.py, so such a scan would either whitelist
|
||||
# the very string it is looking for or cry wolf on every export.
|
||||
_CONTENT_PREFIXES = ("kb/", "raw/")
|
||||
_CONTENT_ALLOWED_NAMES = ("CONTRACT.md", "COLLECTION.md", "log.md", ".gitkeep")
|
||||
|
||||
|
||||
def find_leaks(plan: dict[str, PlannedFile]) -> list[str]:
|
||||
"""Planned paths that carry one instance's own data instead of machinery."""
|
||||
leaks: list[str] = []
|
||||
for relative in sorted(plan):
|
||||
name = relative.rsplit("/", 1)[-1]
|
||||
if name in config.PERSONALIZATION_FILES or name == config.ENVIRONMENT_FILE:
|
||||
leaks.append(f"{relative} (one instance's own personalization)")
|
||||
elif relative.startswith("instructions/dev/"):
|
||||
leaks.append(f"{relative} (stack-development only)")
|
||||
elif relative.startswith(_CONTENT_PREFIXES) and name not in _CONTENT_ALLOWED_NAMES:
|
||||
leaks.append(f"{relative} (wiki content, not machinery)")
|
||||
return leaks
|
||||
|
||||
|
||||
def _write_plan(target: Path, plan: dict[str, PlannedFile]) -> None:
|
||||
for relative, planned in plan.items():
|
||||
dest = target / relative
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
if isinstance(planned.content, bytes):
|
||||
dest.write_bytes(planned.content)
|
||||
else:
|
||||
dest.write_text(planned.content, encoding="utf-8")
|
||||
if planned.executable:
|
||||
dest.chmod(dest.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
|
||||
@app.command("export")
|
||||
def export_command(
|
||||
target: Path = typer.Argument(
|
||||
..., help="Directory to write the distribution into. Must not exist, or must be empty."
|
||||
),
|
||||
dry_run: bool = typer.Option(
|
||||
False, "--dry-run", help="List what would be written, without writing anything."
|
||||
),
|
||||
source_repo: Optional[str] = typer.Option(
|
||||
None, "--source-repo", help="Repository this export was built from (recorded in the stamp)"
|
||||
),
|
||||
source_commit: Optional[str] = typer.Option(
|
||||
None, "--source-commit", help="Commit this export was built from (recorded in the stamp)"
|
||||
),
|
||||
release_url: Optional[str] = typer.Option(
|
||||
None, "--release-url", help="Release page this export ships as (recorded in the stamp)"
|
||||
),
|
||||
update_url: Optional[str] = typer.Option(
|
||||
None, "--update-url", help="Release feed `version check` should ask (recorded in the stamp)"
|
||||
),
|
||||
):
|
||||
"""Export a contentless, distributable copy of this repo's machinery:
|
||||
AGENTS.md/README.md (dev-instance-only marker blocks removed),
|
||||
instructions/ (no instructions/dev/), types/, tools/ (no venv/caches),
|
||||
the .github/hooks/+.vibe session-tracing config plus .claude/settings.json,
|
||||
every kb/*/COLLECTION.md (no pages, no areas), empty
|
||||
raw/{articles,documents,notes,assets}/, VERSION, the USER.md/SOUL.md
|
||||
personalization templates (never the filled files), and a
|
||||
.wikitool-release.json stamp. The --source-*/--release-url/--update-url
|
||||
options only fill fields in that stamp: `export` never calls git and cannot
|
||||
discover them. See instructions/setup-instance.md for what comes next."""
|
||||
run_export(
|
||||
target,
|
||||
dry_run=dry_run,
|
||||
origin=Origin(
|
||||
source_repo=source_repo,
|
||||
source_commit=source_commit,
|
||||
release_url=release_url,
|
||||
update_url=update_url,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def run_export(target: Path, dry_run: bool = False, origin: Optional[Origin] = None) -> None:
|
||||
"""The export itself, free of Typer's option objects so it can be called
|
||||
directly - by the command above, and by the tests."""
|
||||
target = target.resolve()
|
||||
if target.exists():
|
||||
if not target.is_dir():
|
||||
fail(f"{target} exists and is not a directory.")
|
||||
if any(target.iterdir()):
|
||||
fail(f"{target} is not empty. `dist export` refuses to write into a non-empty directory.")
|
||||
|
||||
try:
|
||||
plan = build_plan(origin)
|
||||
except version_mod.VersionError as exc:
|
||||
fail(f"{exc} - a distribution must carry the version it ships.")
|
||||
return
|
||||
|
||||
leaks = find_leaks(plan)
|
||||
if leaks:
|
||||
fail(
|
||||
"export would carry this instance's own data, not just machinery:\n "
|
||||
+ "\n ".join(leaks)
|
||||
+ "\nThis is an allowlist bug in dist_cmd.py, not something to work "
|
||||
"around - fix the allowlist rather than deleting files from the target."
|
||||
)
|
||||
return
|
||||
|
||||
if dry_run:
|
||||
for relative in sorted(plan):
|
||||
typer.echo(f"write {relative}")
|
||||
success(f"Dry run: would write {len(plan)} file(s) to {target}. Nothing written.")
|
||||
return
|
||||
|
||||
_write_plan(target, plan)
|
||||
success(f"Exported {len(plan)} file(s) to {rel_path(target)}.")
|
||||
Reference in New Issue
Block a user