Chemenu 2.1.0 - deterministischer Wissenskompiler
Chemenu kompiliert Rohnotizen zu einem verlinkten, quellengebundenen Wiki: raw/ -> types/ + tools/ -> kb/ -> reports/. Was mechanisch ist, macht tools/wikitool; was Urteil braucht, macht ein Agent unter Contracts, deren Grenzen in Code durchgesetzt sind statt im Prompt. Dieser Commit ist der Startpunkt der oeffentlichen Historie. Die vorherige Entwicklung fand in einer privaten Instanz statt und ist nicht Teil dieses Repositorys; ihre Erzaehlung steht vollstaendig in CHANGES.md, das mit 44 Eintraegen von 0.1.0 bis 2.1.0 erhalten geblieben ist. Der mitgelieferte Korpus ist ein Testbett und eine Demo: 170 Seiten ueber den Stack selbst - Gates, Lint, Versionierung, Suche, das Wiki-Muster. Er dokumentiert das Werkzeug mit den eigenen Mitteln des Werkzeugs. Lizenz: AGPL-3.0 fuer den Stack (tools/, types/), CC-BY-4.0 fuer die Inhalte. Die Grenze zwischen beiden ist der Dateiplan, den dist export berechnet - siehe NOTICE.
This commit is contained in:
torben committed
commit
18ae28f918
368 files changed
+50628
No files matched your search
@@ -0,0 +1,184 @@
|
||||
"""`wikitool sources ...` - raw-file <-> wiki provenance tooling.
|
||||
|
||||
This is the deterministic backbone for citation backtracing: it never guesses
|
||||
which raw file backs a claim, it only reports what the frontmatter and inline
|
||||
`[^cite-id]` footnotes already declare. Filling in those declarations
|
||||
correctly is still the LLM's job.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
|
||||
from chemenu import config
|
||||
from chemenu.commands._util import fail, rel_path, success
|
||||
from chemenu.provenance import (
|
||||
broken_raw_refs,
|
||||
citing_pages,
|
||||
legacy_source_pages,
|
||||
page_raw_files,
|
||||
source_pages_by_raw_file,
|
||||
source_raw_files,
|
||||
uncovered_raw_files,
|
||||
)
|
||||
from chemenu.kb_scan import load_kb_pages
|
||||
|
||||
app = typer.Typer(help="Trace and lint raw-file <-> wiki-page provenance.")
|
||||
|
||||
|
||||
def _normalize_raw_path(raw: str) -> str:
|
||||
"""Accept an absolute path or a path relative to the repo root or to raw/,
|
||||
and return it as a path relative to the repo root (matching how it is
|
||||
stored in `raw_files:`)."""
|
||||
candidate = Path(raw)
|
||||
if candidate.is_absolute():
|
||||
try:
|
||||
return str(candidate.relative_to(config.ROOT))
|
||||
except ValueError:
|
||||
return str(candidate)
|
||||
if candidate.exists():
|
||||
return str(candidate)
|
||||
if (config.ROOT / candidate).exists():
|
||||
return str(candidate)
|
||||
if (config.RAW_DIR / candidate).exists():
|
||||
return str((Path("raw") / candidate))
|
||||
return raw
|
||||
|
||||
|
||||
@app.command("coverage")
|
||||
def coverage(json_out: bool = typer.Option(False, "--json", help="Print raw findings as JSON")):
|
||||
"""Report raw files with no source page, broken raw_files: references, and
|
||||
source pages still using a legacy directory/URL-only `source:` field."""
|
||||
pages = load_kb_pages(config.KB_DIR)
|
||||
report = {
|
||||
"uncovered_raw_files": uncovered_raw_files(config.RAW_DIR, pages),
|
||||
"broken_raw_refs": broken_raw_refs(pages),
|
||||
"legacy_source_pages": legacy_source_pages(pages),
|
||||
}
|
||||
if json_out:
|
||||
typer.echo(json.dumps(report, indent=2))
|
||||
return
|
||||
|
||||
typer.echo(f"Uncovered raw files: {len(report['uncovered_raw_files'])}")
|
||||
for f in report["uncovered_raw_files"]:
|
||||
typer.echo(f" - {f}")
|
||||
typer.echo(f"Broken raw_files references: {len(report['broken_raw_refs'])}")
|
||||
for item in report["broken_raw_refs"]:
|
||||
typer.echo(f" - [[{item['page']}]] -> {item['raw_path']}")
|
||||
typer.echo(f"Legacy (directory/URL-only) source pages: {len(report['legacy_source_pages'])}")
|
||||
for item in report["legacy_source_pages"]:
|
||||
typer.echo(f" - [[{item['page']}]] ({item['reason']}): {item['source']}")
|
||||
|
||||
|
||||
@app.command("trace")
|
||||
def trace(
|
||||
raw: Optional[str] = typer.Option(None, "--raw", help="Raw file path to trace forward from"),
|
||||
page: Optional[str] = typer.Option(None, "--page", help="Wiki page title to trace backward from"),
|
||||
):
|
||||
"""Trace provenance in either direction: --raw shows which source pages
|
||||
cover a raw file and which wiki pages cite it; --page shows which sources
|
||||
and raw files back a given wiki page."""
|
||||
if bool(raw) == bool(page):
|
||||
fail("Provide exactly one of --raw or --page")
|
||||
|
||||
pages = load_kb_pages(config.KB_DIR)
|
||||
|
||||
if raw:
|
||||
raw_key = _normalize_raw_path(raw)
|
||||
by_raw = source_pages_by_raw_file(pages)
|
||||
source_titles = by_raw.get(raw_key, [])
|
||||
if not source_titles:
|
||||
typer.echo(f"No source page covers {raw_key}")
|
||||
raise typer.Exit(code=1)
|
||||
for source_title in source_titles:
|
||||
typer.echo(f"{raw_key}")
|
||||
typer.echo(f" covered by: [[{source_title}]]")
|
||||
citers = citing_pages(pages, source_title)
|
||||
if citers:
|
||||
for c in citers:
|
||||
typer.echo(f" cited by: [[{c}]]")
|
||||
else:
|
||||
typer.echo(" cited by: (nothing yet)")
|
||||
return
|
||||
|
||||
target_page = pages.get(page)
|
||||
if target_page is None:
|
||||
fail(f"No page titled '{page}' found")
|
||||
sources = target_page.frontmatter.get("sources") or []
|
||||
typer.echo(f"[[{page}]]")
|
||||
if not sources:
|
||||
typer.echo(" sources: (none listed)")
|
||||
for source_title in sources:
|
||||
typer.echo(f" sources: [[{source_title}]]")
|
||||
source_page = pages.get(source_title)
|
||||
if source_page is None:
|
||||
typer.echo(" (source page not found)")
|
||||
continue
|
||||
for raw_path in source_raw_files(source_page):
|
||||
typer.echo(f" raw file: {raw_path}")
|
||||
raw_files = page_raw_files(pages, target_page)
|
||||
typer.echo(f" all raw files (incl. inline citations): {raw_files or '(none)'}")
|
||||
|
||||
|
||||
def build_provenance_index(kb_dir: Path, raw_dir: Path) -> str:
|
||||
pages = load_kb_pages(kb_dir)
|
||||
by_raw = source_pages_by_raw_file(pages)
|
||||
all_raw = sorted(str(p.relative_to(config.ROOT)) for p in config.iter_raw_files(raw_dir))
|
||||
uncovered = uncovered_raw_files(raw_dir, pages)
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append("# Provenance Index")
|
||||
lines.append("")
|
||||
lines.append("Generated by `tools/wikitool sources rebuild-index`. Do not hand-edit.")
|
||||
lines.append("")
|
||||
lines.append("Maps every raw source file to the wiki source page(s) that cover it, and")
|
||||
lines.append("every wiki page that cites that source (via frontmatter `sources:` or an")
|
||||
lines.append("inline `[^cite-id]` footnote).")
|
||||
lines.append("")
|
||||
lines.append("## Coverage Summary")
|
||||
lines.append("")
|
||||
lines.append(f"- **Total raw files:** {len(all_raw)}")
|
||||
lines.append(f"- **Covered:** {len(all_raw) - len(uncovered)}")
|
||||
lines.append(f"- **Uncovered:** {len(uncovered)}")
|
||||
lines.append("")
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
lines.append("## Raw Files")
|
||||
lines.append("")
|
||||
|
||||
for raw_path in all_raw:
|
||||
lines.append(f"### `{raw_path}`")
|
||||
lines.append("")
|
||||
source_titles = by_raw.get(raw_path, [])
|
||||
if not source_titles:
|
||||
lines.append("No source page covers this file yet.")
|
||||
lines.append("")
|
||||
continue
|
||||
for source_title in source_titles:
|
||||
lines.append(f"- Covered by: [[{source_title}]]")
|
||||
citers = citing_pages(pages, source_title)
|
||||
if citers:
|
||||
lines.append(f" - Cited by: {', '.join(f'[[{c}]]' for c in citers)}")
|
||||
else:
|
||||
lines.append(" - Cited by: (nothing yet)")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
@app.command("rebuild-index")
|
||||
def rebuild_index(
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="Print the result instead of writing wiki/provenance.md"),
|
||||
):
|
||||
content = build_provenance_index(config.KB_DIR, config.RAW_DIR)
|
||||
provenance_file = config.KB_DIR / "provenance.md"
|
||||
if dry_run:
|
||||
# nl=False so the preview is byte-identical to the file that would be
|
||||
# written; see the same note in index_build.py.
|
||||
typer.echo(content, nl=False)
|
||||
return
|
||||
provenance_file.write_text(content, encoding="utf-8")
|
||||
success(f"Rebuilt {rel_path(provenance_file)}")
|
||||
Reference in new issue
Block a user