converters: Testkorpus und Harness ziehen nach torben/chemenu-testfiles, Workflow Converters mit Pfad-Trigger, Release-Gate (#232); 10.1.0-beta.2

Files changed: - .gitattributes - .gitea/workflows/converters.yml - .gitea/workflows/release.yml - .gitignore - CHANGES.md - VERSION - docs/dev/development.md - instructions/dev/dev-setup.md - instructions/dev/publish-and-ci.md - instructions/dev/testing-conventions.md - testfiles/README.md - testfiles/mail/github-outlook.msg - testfiles/mail/github-roundcube.eml - testfiles/mail/gog-outlook.msg - testfiles/mail/gog-roundcube.eml - testfiles/mail/gog-roundcube.eml.expect - testfiles/mail/tldr-outlook.msg - testfiles/mail/tldr-roundcube.eml - testfiles/office/excel-artificial.xlsx - testfiles/office/excel-business.xlsx - testfiles/office/excel-business.xlsx.expect - testfiles/office/excel-science.xlsx - testfiles/office/pptx-baseline.pptx - testfiles/office/pptx-master.pptx - testfiles/office/pptx-objects.pptx - testfiles/office/word-complex.docx - testfiles/office/word-simple.docx - testfiles/pdf/einstein-dissertation.pdf - testfiles/pdf/ifg.pdf - testfiles/pdf/nist-ai-rmf.pdf - testfiles/pdf/nist-ssdf-genai.pdf - testfiles/zip/archive.zip - tools/chemenu/tests/test_raw_derive_testfiles.py Co-Authored-By: Claude Code <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
2026-10-14 15:05:12 +02:00
1 parent 3bf179ae8c
commit 71170bc1e8
@@ -0,0 +1,87 @@
+# The converters against real documents.
+#
+# `ci.yml` proves the filters on documents built to be easy (`test_raw_derive.py`,
+# `test_raw_derive_binary.py`). This workflow derives the real ones: the test corpus and its
+# harness live in the private repository `torben/chemenu-testfiles`, cloned into `testfiles/`
+# (gitignored in this repository) and run with this checkout's `tools/.venv`. The corpus is large
+# (some 76 MB, one book needs 48 s per derivation) and holds material that may not sit in a public
+# repository, which is why none of it runs in the standard CI.
+#
+# When it runs:
+# - on a push to `main` that touches the converter surface (the `paths:` list below). Gitea
+#   evaluates `paths:` the way GitHub does - observed for `paths-ignore` in `ci.yml` and in use for
+#   `paths:` in `release.yml`; do not re-derive it;
+# - by hand (`workflow_dispatch`), which is the way to run it after a change in the private
+#   repository alone: a push there starts nothing here, and a commit that reaches the converters
+#   only indirectly is not in the list;
+# - before a release without suffix: `release.yml` carries the same steps as its job `converters`
+#   and the release job waits for it. A candidate (`-beta.N`) does not run it.
+#
+# It always tests `main` of the private repository. A converter change that alters an expected
+# output (`.expect`) is therefore two commits, one per repository, the private one first.
+#
+# Runner shape: identical to ci.yml, and for the same reason - see its header for `nodejs`.
+#
+# Access to the private repository: the per-job token (`${{ gitea.token }}`), as `release.yml`
+# uses it for this repository. If this instance does not let it read a second, private repository
+# of the same owner, the clone step fails with an authentication error; the way out is a read token
+# in 1Password, loaded with `1password/load-secrets-action` the way `ci.yml` loads its secret
+# reference (the service account sees only the CI-CD vault).
+
+name: Converters
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'tools/chemenu/filters/**'
+      - 'tools/chemenu/commands/raw/derive.py'
+      - 'tools/chemenu/commands/raw/accept.py'
+      - 'tools/chemenu/tests/test_raw_derive*.py'
+      - 'tools/chemenu/tests/binary_formats.py'
+      - 'tools/requirements.txt'
+      - '.gitea/workflows/converters.yml'
+  workflow_dispatch:
+
+jobs:
+  converters:
+    runs-on: linux-docker
+    timeout-minutes: 30
+    container:
+      image: debian:trixie-slim
+    env:
+      WIKITOOL_SESSION_ID: converters-${{ github.run_id }}
+      WIKI_TRACE_DIR: /tmp/wikitool-trace
+
+    steps:
+      - name: System dependencies
+        run: |
+          set -eu
+          apt-get update -qq
+          apt-get install -y --no-install-recommends \
+            python3 python3-venv git nodejs ripgrep ca-certificates
+          rm -rf /var/lib/apt/lists/*
+
+      - uses: actions/checkout@v7
+
+      - name: The private test corpus
+        uses: actions/checkout@v7
+        with:
+          repository: torben/chemenu-testfiles
+          token: ${{ gitea.token }}
+          path: testfiles
+
+      - name: Tool environment
+        run: |
+          set -eu
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          tools/preflight.sh
+          tools/.venv/bin/python -m pip install --quiet pytest
+
+      - name: Real documents
+        # `-c pytest.ini` applies the suite's own configuration to a harness that lives outside
+        # `tools/`; the harness conftest pulls in the suite's conftest (testfiles/README.md).
+        run: |
+          set -eu
+          cd tools
+          .venv/bin/python -m pytest -c pytest.ini ../testfiles/harness -q --durations=5
@@ -24,6 +24,13 @@
 # tag). It is a separate job so that a failing mirror leaves the release and its
 # tag standing: the release is the product, the wiki a view of it. It does not
 # run for a `-beta.N` candidate, for which `release` sets `skip`.
+#
+# `converters` is the first job: the converter tests against the real documents of the private
+# repository `torben/chemenu-testfiles` (converters.yml has the reasoning and is the reference for
+# the steps; the two are kept in step by hand, because this instance's support for calling one
+# workflow from another is not something this repository relies on). `release` waits for it with
+# `needs:`, so a red harness stops the run before a release or a tag exists. For a candidate it
+# decides `skip` the way `release` does and ends after its first steps.

 name: Release

@@ -39,7 +46,65 @@ on:
  workflow_dispatch:

 jobs:
+  converters:
+    runs-on: linux-docker
+    timeout-minutes: 30
+    container:
+      image: debian:trixie-slim
+    env:
+      WIKITOOL_SESSION_ID: release-converters-${{ github.run_id }}
+      WIKI_TRACE_DIR: /tmp/wikitool-trace
+
+    steps:
+      - name: System dependencies
+        run: |
+          set -eu
+          apt-get update -qq
+          apt-get install -y --no-install-recommends \
+            python3 python3-venv git nodejs ripgrep ca-certificates
+          rm -rf /var/lib/apt/lists/*
+
+      - uses: actions/checkout@v7
+
+      - name: A candidate needs no converter run
+        id: version
+        run: |
+          set -eu
+          case "$(tr -d '[:space:]' < VERSION)" in
+            *-beta.*)
+              echo "VERSION is a running candidate - the converter tests run for a release only."
+              echo "skip=true" >> "$GITHUB_OUTPUT"
+              ;;
+            *)
+              echo "skip=false" >> "$GITHUB_OUTPUT"
+              ;;
+          esac
+
+      - name: The private test corpus
+        if: steps.version.outputs.skip != 'true'
+        uses: actions/checkout@v7
+        with:
+          repository: torben/chemenu-testfiles
+          token: ${{ gitea.token }}
+          path: testfiles
+
+      - name: Tool environment
+        if: steps.version.outputs.skip != 'true'
+        run: |
+          set -eu
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          tools/preflight.sh
+          tools/.venv/bin/python -m pip install --quiet pytest
+
+      - name: Real documents
+        if: steps.version.outputs.skip != 'true'
+        run: |
+          set -eu
+          cd tools
+          .venv/bin/python -m pytest -c pytest.ini ../testfiles/harness -q --durations=5
+
  release:
+    needs: converters
    runs-on: linux-docker
    container:
      image: debian:trixie-slim