raw: fetch behind a login - raw login stores the user's session as a secret outside the tree, fetch.routes sends a host through it or to an external fetcher, a handler's render goes through the optional browser component (preflight --with browser) (#169); 10.0.0-beta.8
Files changed: - CHANGES.md - INSTALL.md - VERSION - docs/how-to/README.md - docs/how-to/fetch-pages-behind-a-login.md - docs/how-to/ingest-a-source.md - docs/reference/commands/README.md - docs/reference/commands/instance-health.md - docs/reference/commands/raw-material-and-uploads.md - docs/reference/configuration.md - docs/reference/exit-codes.md - instructions/preflight.md - instructions/wiki-ingest/SKILL.md - raw/CONTRACT.md - sites/CONTRACT.md - tools/CONTRACT.md - tools/README.md - tools/chemenu/browser.py - tools/chemenu/cli_contract.py - tools/chemenu/commands/doctor.py - tools/chemenu/commands/raw/__init__.py - tools/chemenu/commands/raw/fetch.py - tools/chemenu/commands/raw/login.py - tools/chemenu/config_shapes.py - tools/chemenu/fetch_routes.py - tools/chemenu/reference_doc.py - tools/chemenu/settings.py - tools/chemenu/sites/__init__.py - tools/chemenu/sites/data.py - tools/chemenu/sites/site.schema.json - tools/chemenu/tests/test_cli.py - tools/chemenu/tests/test_config_shapes.py - tools/chemenu/tests/test_doctor.py - tools/chemenu/tests/test_preflight.py - tools/chemenu/tests/test_preflight_pwsh.py - tools/chemenu/tests/test_raw_fetch_routes.py - tools/chemenu/tests/test_raw_login.py - tools/chemenu/tests/test_site_files.py - tools/chemenu/web_capture.py - tools/chemenu/web_session.py - tools/preflight.ps1 - tools/preflight.sh - tools/requirements-browser.txt Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SnAJ7Z3CpVD3PRbN73QtU2
This commit is contained in:
torbenandClaude Opus 5.5 committed
1 parent
aed846740b
commit
5994b2a88c
43 files changed
+2189
-75
No files matched your search
+84
-4
@@ -4,6 +4,12 @@
|
||||
#
|
||||
# pwsh -NoProfile -ExecutionPolicy Bypass -File tools/preflight.ps1
|
||||
# pwsh -NoProfile -ExecutionPolicy Bypass -File tools/preflight.ps1 --set rg=C:\Tools\rg.exe
|
||||
# pwsh -NoProfile -ExecutionPolicy Bypass -File tools/preflight.ps1 --with browser
|
||||
#
|
||||
# --with browser installs tools/requirements-browser.txt into tools/.venv and the
|
||||
# Chromium that Playwright drives into tools/.venv/ms-playwright - only on request,
|
||||
# because it is a large download that only `raw login` and pages a site handler
|
||||
# declares `render` need. Once installed, every later run keeps it up to date.
|
||||
#
|
||||
# As the release asset `preflight.ps1` - a copy with no tools/prerequisites.txt next
|
||||
# to it - the script is the first install step instead: it downloads the stack
|
||||
@@ -57,6 +63,9 @@ $ToolsFile = Join-Path $Root '.wikitool-tools.json'
|
||||
$Venv = Join-Path $Dir '.venv'
|
||||
$Requirements = Join-Path $Dir 'requirements.txt'
|
||||
$Stamp = Join-Path $Venv '.chemenu-requirements.sha256'
|
||||
$BrowserRequirements = Join-Path $Dir 'requirements-browser.txt'
|
||||
$BrowserStamp = Join-Path $Venv '.chemenu-browser.sha256'
|
||||
$Browsers = Join-Path $Venv 'ms-playwright'
|
||||
$Self = 'pwsh -NoProfile -ExecutionPolicy Bypass -File tools/preflight.ps1'
|
||||
|
||||
# Filled in by the release build, in the copy attached to the release; empty in a tree.
|
||||
@@ -80,11 +89,30 @@ function Write-ErrorLine {
|
||||
$Sets = @{}
|
||||
$Into = ''
|
||||
$Archive = ''
|
||||
$With = ''
|
||||
$index = 0
|
||||
while ($index -lt $args.Count) {
|
||||
$arg = [string]$args[$index]
|
||||
$given = $null
|
||||
if ($arg -eq '--set') {
|
||||
if ($arg -eq '--with' -or $arg.StartsWith('--with=')) {
|
||||
if ($arg -eq '--with') {
|
||||
if ($index + 1 -ge $args.Count -or -not [string]$args[$index + 1]) {
|
||||
Write-ErrorLine 'preflight: --with needs a component name (known: browser)'
|
||||
exit 1
|
||||
}
|
||||
$index++
|
||||
$component = [string]$args[$index]
|
||||
} else {
|
||||
$component = $arg.Substring(7)
|
||||
}
|
||||
if ($component -ne 'browser') {
|
||||
Write-ErrorLine "preflight: --with names '$component', which is not an optional component (known: browser)"
|
||||
exit 1
|
||||
}
|
||||
$With = $component
|
||||
$index++
|
||||
continue
|
||||
} elseif ($arg -eq '--set') {
|
||||
if ($index + 1 -ge $args.Count) {
|
||||
Write-ErrorLine 'preflight: --set needs <tool>=<path>'
|
||||
exit 1
|
||||
@@ -107,12 +135,13 @@ while ($index -lt $args.Count) {
|
||||
$index++
|
||||
continue
|
||||
} elseif ($arg -eq '-h' -or $arg -eq '--help') {
|
||||
Write-Line "usage: $Self [--set <tool>=<path>]..."
|
||||
Write-Line ' preflight.ps1 [--into <path>] [--archive <tarball>] [--set <tool>=<path>]...'
|
||||
Write-Line "usage: $Self [--set <tool>=<path>]... [--with browser]"
|
||||
Write-Line ' preflight.ps1 [--into <path>] [--archive <tarball>] [--set <tool>=<path>]... [--with browser]'
|
||||
Write-Line ''
|
||||
Write-Line 'Checks the tools this stack needs (tools/prerequisites.txt), records their paths'
|
||||
Write-Line 'in .wikitool-tools.json and sets up tools/.venv. Exit 0 means ready; exit 42'
|
||||
Write-Line 'means the user has to act - the output says how.'
|
||||
Write-Line 'means the user has to act - the output says how. --with browser also installs'
|
||||
Write-Line 'the optional browser component (Playwright and its Chromium) into tools/.venv.'
|
||||
Write-Line ''
|
||||
Write-Line 'The second form is the release asset: it downloads the release (or takes'
|
||||
Write-Line '--archive), checks its sha256, unpacks it into its own folder or --into (empty,'
|
||||
@@ -492,6 +521,9 @@ function Invoke-AssetMode {
|
||||
foreach ($name in $Sets.Keys) {
|
||||
$passed += "--set=$name=$($Sets[$name])"
|
||||
}
|
||||
if ($With) {
|
||||
$passed += "--with=$With"
|
||||
}
|
||||
& ([Environment]::ProcessPath) -NoProfile -ExecutionPolicy Bypass -File $treeScript @passed
|
||||
exit $LASTEXITCODE
|
||||
}
|
||||
@@ -1068,6 +1100,54 @@ if ($want -ne $have) {
|
||||
Write-Line ('OK venv - {0}' -f $Venv)
|
||||
|
||||
Write-ToolsFile $true
|
||||
|
||||
# --- optional components: installed on request, then kept up to date ---------------
|
||||
# After .wikitool-tools.json is complete on purpose: a component that fails to install
|
||||
# stops this run with exit 42, but leaves every wikitool command working.
|
||||
|
||||
function Test-Browser {
|
||||
if (-not (Test-Path -LiteralPath $Browsers -PathType Container)) {
|
||||
return $false
|
||||
}
|
||||
return $null -ne (Get-ChildItem -LiteralPath $Browsers -Filter 'chromium*' -ErrorAction SilentlyContinue | Select-Object -First 1)
|
||||
}
|
||||
|
||||
if ($With -eq 'browser' -or (Test-Path -LiteralPath $BrowserStamp -PathType Leaf)) {
|
||||
if (-not (Test-Path -LiteralPath $BrowserRequirements -PathType Leaf)) {
|
||||
Write-ErrorLine "preflight: $BrowserRequirements is missing - the stack tree next to this script is incomplete."
|
||||
exit 1
|
||||
}
|
||||
$want = (Get-FileHash -LiteralPath $BrowserRequirements -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$have = ''
|
||||
if (Test-Path -LiteralPath $BrowserStamp -PathType Leaf) {
|
||||
$have = (Get-Content -Raw -LiteralPath $BrowserStamp).Trim()
|
||||
}
|
||||
if ($want -ne $have -or -not (Test-Browser)) {
|
||||
$installed = Invoke-NativeVerbose -Path $venvPython -Arguments @('-m', 'pip', 'install', '--disable-pip-version-check', '--quiet', '-r', $BrowserRequirements)
|
||||
if ($installed.Code -ne 0) {
|
||||
Add-Problem 'The optional browser component could not be installed into tools/.venv.' `
|
||||
'only raw login and pages a site handler fetches through a browser need it; everything else works without it' `
|
||||
"Check that this computer can reach the internet (a proxy or a security program can block it; if so, ask whoever looks after this computer), then run: $Self --with browser`npip said:`n$(Get-LastLine $installed.Output)"
|
||||
Exit-WithGuide
|
||||
}
|
||||
$previous = $env:PLAYWRIGHT_BROWSERS_PATH
|
||||
$env:PLAYWRIGHT_BROWSERS_PATH = $Browsers
|
||||
try {
|
||||
$downloaded = Invoke-NativeVerbose -Path $venvPython -Arguments @('-m', 'playwright', 'install', 'chromium')
|
||||
} finally {
|
||||
$env:PLAYWRIGHT_BROWSERS_PATH = $previous
|
||||
}
|
||||
if ($downloaded.Code -ne 0) {
|
||||
Add-Problem 'The browser of the optional browser component could not be downloaded.' `
|
||||
'only raw login and pages a site handler fetches through a browser need it; everything else works without it' `
|
||||
"Check that this computer can reach the internet, then run: $Self --with browser`nPlaywright said:`n$(Get-LastLine $downloaded.Output)"
|
||||
Exit-WithGuide
|
||||
}
|
||||
[IO.File]::WriteAllText($BrowserStamp, "$want`n", [Text.UTF8Encoding]::new($false))
|
||||
}
|
||||
Write-Line ('OK browser - {0}' -f $Browsers)
|
||||
}
|
||||
|
||||
Write-Line ''
|
||||
Write-Line 'Preflight passed. Tool paths are recorded in .wikitool-tools.json; tools/wikitool is ready.'
|
||||
exit 0
|
||||
Reference in new issue
Block a user