testfiles: real documents get short names and a source table, personal data is replaced by same-length filler (#203)
Adds the nineteen real documents under testfiles/ (mail, Office, PDF, ZIP) with short ASCII names. Recipient addresses, names, host names, mailbox identifiers and recipient-bound tokens are replaced by filler of the same length, and the archive is rebuilt from the sanitized members. test_raw_derive_testfiles.py now compares the sheet TSVs too (with a direct test on _texts()), and a real workbook carries a formula needle that only a .formulas.tsv holds. testfiles/README.md gains the provenance table and the two new rules. Files changed: - CHANGES.md - VERSION - testfiles/README.md - testfiles/mail/github-outlook.msg - testfiles/mail/github-roundcube.eml - testfiles/mail/gog-outlook.msg - testfiles/mail/gog-roundcube.eml - testfiles/mail/tldr-outlook.msg - testfiles/mail/tldr-roundcube.eml - testfiles/office/excel-artificial.xlsx - testfiles/office/excel-business.xlsx - testfiles/office/excel-business.xlsx.expect - testfiles/office/excel-science.xlsx - testfiles/office/pptx-baseline.pptx - testfiles/office/pptx-master.pptx - testfiles/office/pptx-objects.pptx - testfiles/office/word-complex.docx - testfiles/office/word-simple.docx - testfiles/pdf/einstein-dissertation.pdf - testfiles/pdf/ifg.pdf - testfiles/pdf/nist-ai-rmf.pdf - testfiles/pdf/nist-ssdf-genai.pdf - testfiles/zip/archive.zip - tools/chemenu/tests/test_raw_derive_testfiles.py Co-Authored-By: Claude Code <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
This commit is contained in:
1 parent
9868fce3d5
commit
be049217a9
24 files changed
+2359
-5
No files matched your search
@@ -55,6 +55,8 @@ def _texts(folder: Path) -> dict[str, str]:
|
||||
texts[md.relative_to(folder).as_posix()] = "\n".join(
|
||||
line for line in lines if not line.startswith("derived:")
|
||||
)
|
||||
for tsv in sorted(folder.rglob("*.sheets/*.tsv")):
|
||||
texts[tsv.relative_to(folder).as_posix()] = tsv.read_text(encoding="utf-8")
|
||||
return texts
|
||||
|
||||
|
||||
@@ -102,3 +104,17 @@ def test_a_real_document_derives_stably_and_is_accepted(tree, document):
|
||||
page=None, replaces=None, dry_run=False,
|
||||
)
|
||||
assert not (tree / "incoming" / document.stem).exists()
|
||||
|
||||
|
||||
def test_texts_covers_the_sheet_tsvs_beside_the_markdown(tmp_path):
|
||||
(tmp_path / "x.sheets").mkdir()
|
||||
(tmp_path / "derived.md").write_text("derived: now\nbody\n", encoding="utf-8")
|
||||
cells = tmp_path / "x.sheets" / "01-a.tsv"
|
||||
cells.write_text("a\tb\n", encoding="utf-8")
|
||||
|
||||
before = _texts(tmp_path)
|
||||
assert before["derived.md"] == "body"
|
||||
assert before["x.sheets/01-a.tsv"] == "a\tb\n"
|
||||
|
||||
cells.write_text("a\tc\n", encoding="utf-8")
|
||||
assert _texts(tmp_path) != before
|
||||
Reference in new issue
Block a user