mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-14 13:05:09 +00:00
chore(scripts): expand bench suite for cross-machine runs
Adds machine info to the report header, multiprocess and strided-split runs, hedge/slow-tail cases, the rnet path, and the leak regression tests; ps1 reports save as utf8 and expected failures no longer render as NativeCommandError blocks.
This commit is contained in:
imSp4rky
@@ -0,0 +1,12 @@
|
||||
@echo off
|
||||
REM Run the downloader benchmark suite and save a readable report for comparison.
|
||||
REM Offline synthetic + fault runs by default; --live also hits public test CDNs
|
||||
REM (needs network + ffprobe on PATH).
|
||||
REM scripts\bench_suite.bat (offline suite)
|
||||
REM scripts\bench_suite.bat --live (+ live/network tests)
|
||||
REM Output: scripts\bench-results\<host>_<timestamp>.txt
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
if not exist "scripts\bench-results" mkdir "scripts\bench-results"
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0bench_suite.ps1" %*
|
||||
endlocal
|
||||
@@ -0,0 +1,106 @@
|
||||
# Run the downloader benchmark suite and save a readable report for comparison.
|
||||
# Offline synthetic + fault runs by default; -live also hits public test CDNs
|
||||
# (needs network + ffprobe on PATH). Usually launched via bench_suite.bat.
|
||||
# powershell -File scripts\bench_suite.ps1 # offline
|
||||
# powershell -File scripts\bench_suite.ps1 --live # + live/network tests
|
||||
param([string]$mode = "offline")
|
||||
$ErrorActionPreference = "Continue"
|
||||
Set-Location (Join-Path $PSScriptRoot "..")
|
||||
|
||||
$live = ($mode -eq "--live" -or $mode -eq "-live" -or $mode -eq "live")
|
||||
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$out = "scripts\bench-results\$($env:COMPUTERNAME)_$stamp.txt"
|
||||
if (-not (Test-Path "scripts\bench-results")) { New-Item -ItemType Directory -Path "scripts\bench-results" | Out-Null }
|
||||
|
||||
function Run($a) {
|
||||
Write-Output ""
|
||||
Write-Output ("==================== python " + ($a -join " ") + " ====================")
|
||||
# stringify the merged stream: PS 5.1 wraps native stderr lines in ErrorRecords,
|
||||
# which render as NativeCommandError blocks (expected-fail runs looked like crashes)
|
||||
& uv run python @a 2>&1 | ForEach-Object { "$_" }
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Output ("(exit code: " + $LASTEXITCODE + ")")
|
||||
}
|
||||
}
|
||||
|
||||
# collect everything, then tee to the report so tables land in the file too
|
||||
& {
|
||||
Write-Output "# unshackle downloader benchmark"
|
||||
Write-Output ("host: " + $env:COMPUTERNAME)
|
||||
Write-Output ("date: " + (Get-Date -Format "yyyy-MM-dd HH:mm:ss"))
|
||||
Write-Output ("git: " + (git rev-parse --abbrev-ref HEAD) + " @ " + (git rev-parse --short HEAD))
|
||||
$cpu = Get-CimInstance Win32_Processor | Select-Object -First 1
|
||||
Write-Output ("cpu: " + $cpu.Name + " (" + $cpu.NumberOfCores + "c/" + $cpu.NumberOfLogicalProcessors + "t)")
|
||||
$mem = [math]::Round((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB)
|
||||
$os = Get-CimInstance Win32_OperatingSystem
|
||||
Write-Output ("mem: " + $mem + " GiB")
|
||||
Write-Output ("os: " + $os.Caption + " " + $os.Version)
|
||||
Write-Output ("python: " + (& uv run python -V 2>&1 | ForEach-Object { "$_" }))
|
||||
Write-Output ("mode: " + $(if ($live) { "live" } else { "offline" }))
|
||||
|
||||
# negotiated link speed per NIC; shows the physical ceiling before the throughput run
|
||||
Write-Output ""
|
||||
Write-Output "==================== nic link speed ===================="
|
||||
Get-NetAdapter | Where-Object Status -eq 'Up' |
|
||||
ForEach-Object { Write-Output ("{0}: {1} ({2})" -f $_.Name, $_.LinkSpeed, $_.InterfaceDescription) }
|
||||
|
||||
# link baseline: max line speed, so downloader numbers can be read against it
|
||||
# (e.g. "2Gbe -> ~1900 Mbps expected ceiling"). Uses whatever CLI exists.
|
||||
Write-Output ""
|
||||
Write-Output "==================== speedtest baseline ===================="
|
||||
if (Get-Command speedtest -ErrorAction SilentlyContinue) {
|
||||
& speedtest --accept-license --accept-gdpr 2>&1 | ForEach-Object { "$_" }
|
||||
} elseif (Get-Command npx -ErrorAction SilentlyContinue) {
|
||||
# Netflix fast.com; download + upload + latency + server, clean when piped
|
||||
& npx --yes fast-cli --upload --verbose 2>&1 | ForEach-Object { "$_" }
|
||||
} elseif (Get-Command speedtest-cli -ErrorAction SilentlyContinue) {
|
||||
& speedtest-cli --simple 2>&1 | ForEach-Object { "$_" }
|
||||
} else {
|
||||
Write-Output "no speedtest CLI found (install Ookla 'speedtest', have npx, or 'pip install speedtest-cli') - skipping"
|
||||
}
|
||||
|
||||
# 1. Synthetic downloader benchmark (offline, deterministic)
|
||||
Run @("scripts/bench_downloader.py", "--segments", "64", "--workers", "1,2,4,8,16", "--runs", "3")
|
||||
Run @("scripts/bench_downloader.py", "--segments", "64", "--workers", "1,2,4,8,16", "--adaptive")
|
||||
Run @("scripts/bench_downloader.py", "--workers", "4,8,16", "--no-read1")
|
||||
|
||||
# 2. Fault paths (--fault-stall is EXPECTED to fail + exit non-zero)
|
||||
Run @("scripts/bench_downloader.py", "--fault-503", "2", "--fast-timeouts", "--segments", "8")
|
||||
Run @("scripts/bench_downloader.py", "--fault-reset", "2", "--fast-timeouts", "--segments", "8")
|
||||
Run @("scripts/bench_downloader.py", "--fault-stall", "1", "--fast-timeouts", "--segments", "8")
|
||||
|
||||
# 3. Multiprocess path (>=24 segments engages spawned children; strided chunk split).
|
||||
# big batch: at small sizes the fixed spawn cost drowns the throughput signal
|
||||
Run @("scripts/bench_downloader.py", "--segments", "128", "--workers", "8", "--procs", "2")
|
||||
Run @("scripts/bench_downloader.py", "--segments", "128", "--workers", "8", "--procs", "4")
|
||||
Run @("scripts/bench_downloader.py", "--fault-stall", "1", "--fast-timeouts", "--segments", "32", "--workers", "8", "--procs", "2")
|
||||
|
||||
# 4. Hedge + slow tail (last segments crawl; hedging and striding bound the tail)
|
||||
Run @("scripts/bench_downloader.py", "--fault-tail-slow", "4", "--fast-timeouts", "--segments", "32", "--workers", "8")
|
||||
Run @("scripts/bench_downloader.py", "--fault-tail-slow", "4", "--fast-timeouts", "--segments", "32", "--workers", "8", "--procs", "4")
|
||||
Run @("scripts/bench_downloader.py", "--fault-tail-slow", "2", "--fast-timeouts", "--adaptive", "--segments", "16", "--workers", "8")
|
||||
|
||||
# 5. rnet session path (loopback through the TLS-fingerprint client's own read loop)
|
||||
Run @("scripts/bench_downloader.py", "--segments", "32", "--workers", "8", "--impersonate", "Chrome131")
|
||||
|
||||
# 6. Leak regression tests (locked handles / leaked workers; unlink checks bite on Windows)
|
||||
Write-Output ""
|
||||
Write-Output "==================== pytest tests/test_downloader_leaks.py ===================="
|
||||
& uv run pytest tests/test_downloader_leaks.py -q 2>&1 | ForEach-Object { "$_" }
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Output "(pytest failed or missing - run 'uv sync' to install dev deps)"
|
||||
}
|
||||
|
||||
# 7. Live network tests (opt-in)
|
||||
if ($live) {
|
||||
Run @("scripts/live_manifest_test.py", "--workers", "16", "--track", "largest")
|
||||
Run @("scripts/live_segmentbase_ab.py", "--track", "largest")
|
||||
Run @("scripts/bench_downloader.py", "--urls-file", "scripts/bench-urls/apple_sustained_4x585mb.txt", "--impersonate", "Chrome131")
|
||||
Run @("scripts/bench_downloader.py", "--urls-file", "scripts/bench-urls/akamai_4k_segments.txt", "--impersonate", "Chrome131")
|
||||
}
|
||||
|
||||
Write-Output ""
|
||||
Write-Output ("report saved: " + $out)
|
||||
} | Tee-Object -Variable report
|
||||
# utf8 (not Tee-Object's utf16) so reports diff and grep cleanly across machines
|
||||
$report | Out-File -FilePath $out -Encoding utf8
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run the downloader benchmark suite and save a readable report for comparison.
|
||||
# Offline synthetic + fault runs by default; --live also hits public test CDNs
|
||||
# (needs network + ffprobe on PATH).
|
||||
# ./scripts/bench_suite.sh # offline suite
|
||||
# ./scripts/bench_suite.sh --live # + live/network tests
|
||||
# Output: scripts/bench-results/<host>_<utc-timestamp>.txt
|
||||
set -uo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
OUT_DIR="scripts/bench-results"
|
||||
mkdir -p "$OUT_DIR"
|
||||
STAMP="$(date -u +%Y%m%d-%H%M%S)"
|
||||
OUT="$OUT_DIR/$(hostname)_${STAMP}.txt"
|
||||
|
||||
# tee everything (stdout+stderr) to the report so tables land in the file too
|
||||
exec > >(tee "$OUT") 2>&1
|
||||
|
||||
echo "# unshackle downloader benchmark"
|
||||
echo "host: $(hostname)"
|
||||
echo "date: $(date -u '+%Y-%m-%d %H:%M:%S') UTC"
|
||||
echo "git: $(git rev-parse --abbrev-ref HEAD) @ $(git rev-parse --short HEAD)"
|
||||
echo "cpu: $(grep -m1 'model name' /proc/cpuinfo | cut -d: -f2- | sed 's/^ *//') ($(nproc) threads)"
|
||||
echo "mem: $(free -h --si 2>/dev/null | awk '/^Mem:/{print $2}')"
|
||||
echo "os: $(uname -sr)"
|
||||
echo "python: $(uv run python -V 2>&1)"
|
||||
echo "mode: ${1:-offline}"
|
||||
|
||||
run() { echo; echo "==================== $* ===================="; uv run python "$@"; }
|
||||
|
||||
# link baseline: establishes the max line speed so downloader numbers can be read
|
||||
# against it (e.g. "2Gbe -> ~1900 Mbps expected ceiling"). Uses whatever CLI exists.
|
||||
# negotiated link speed per NIC (from sysfs); shows the physical ceiling, e.g. 2500 = 2.5GbE
|
||||
nic_link_speed() {
|
||||
echo; echo "==================== nic link speed ===================="
|
||||
local found=0
|
||||
for d in /sys/class/net/*; do
|
||||
local n state spd
|
||||
n=$(basename "$d")
|
||||
[ "$n" = "lo" ] && continue
|
||||
state=$(cat "$d/operstate" 2>/dev/null)
|
||||
[ "$state" = "up" ] || continue
|
||||
spd=$(cat "$d/speed" 2>/dev/null)
|
||||
echo "$n: ${spd:-?} Mbps"
|
||||
found=1
|
||||
done
|
||||
[ "$found" = 1 ] || echo "no up interfaces with readable speed (virtual/WSL/wifi report -1)"
|
||||
}
|
||||
nic_link_speed
|
||||
|
||||
speedtest_baseline() {
|
||||
echo; echo "==================== speedtest baseline ===================="
|
||||
if command -v speedtest >/dev/null 2>&1; then
|
||||
speedtest --accept-license --accept-gdpr 2>&1 || echo "speedtest failed"
|
||||
elif command -v npx >/dev/null 2>&1; then
|
||||
# Netflix fast.com; download + upload + latency + server, clean when piped
|
||||
npx --yes fast-cli --upload --verbose 2>&1 || echo "npx fast-cli failed"
|
||||
elif command -v speedtest-cli >/dev/null 2>&1; then
|
||||
speedtest-cli --simple 2>&1 || echo "speedtest-cli failed"
|
||||
else
|
||||
echo "no speedtest CLI found (install Ookla 'speedtest', have npx, or 'pipx install speedtest-cli') - skipping"
|
||||
fi
|
||||
}
|
||||
speedtest_baseline
|
||||
|
||||
# 1. Synthetic downloader benchmark (offline, deterministic)
|
||||
run scripts/bench_downloader.py --segments 64 --workers 1,2,4,8,16 --runs 3
|
||||
run scripts/bench_downloader.py --segments 64 --workers 1,2,4,8,16 --adaptive
|
||||
run scripts/bench_downloader.py --workers 4,8,16 --no-read1
|
||||
|
||||
# 2. Fault paths (fast timeouts; --fault-stall is EXPECTED to fail + exit non-zero)
|
||||
run scripts/bench_downloader.py --fault-503 2 --fast-timeouts --segments 8
|
||||
run scripts/bench_downloader.py --fault-reset 2 --fast-timeouts --segments 8
|
||||
run scripts/bench_downloader.py --fault-stall 1 --fast-timeouts --segments 8 || true
|
||||
|
||||
# 3. Multiprocess path (>=24 segments engages spawned children; strided chunk split).
|
||||
# big batch: at small sizes the fixed spawn cost drowns the throughput signal
|
||||
run scripts/bench_downloader.py --segments 128 --workers 8 --procs 2
|
||||
run scripts/bench_downloader.py --segments 128 --workers 8 --procs 4
|
||||
run scripts/bench_downloader.py --fault-stall 1 --fast-timeouts --segments 32 --workers 8 --procs 2 || true
|
||||
|
||||
# 4. Hedge + slow tail (last segments crawl; hedging and striding bound the tail)
|
||||
run scripts/bench_downloader.py --fault-tail-slow 4 --fast-timeouts --segments 32 --workers 8
|
||||
run scripts/bench_downloader.py --fault-tail-slow 4 --fast-timeouts --segments 32 --workers 8 --procs 4
|
||||
run scripts/bench_downloader.py --fault-tail-slow 2 --fast-timeouts --adaptive --segments 16 --workers 8
|
||||
|
||||
# 5. rnet session path (loopback through the TLS-fingerprint client's own read loop)
|
||||
run scripts/bench_downloader.py --segments 32 --workers 8 --impersonate Chrome131 || true
|
||||
|
||||
# 6. Leak regression tests (locked handles / leaked workers; unlink checks bite on Windows)
|
||||
echo; echo "==================== pytest tests/test_downloader_leaks.py ===================="
|
||||
uv run pytest tests/test_downloader_leaks.py -q || echo "(pytest failed or missing - run 'uv sync' to install dev deps)"
|
||||
|
||||
# 7. Live network tests (opt-in)
|
||||
if [[ "${1:-}" == "--live" ]]; then
|
||||
run scripts/live_manifest_test.py --workers 16 --track largest
|
||||
run scripts/live_segmentbase_ab.py --track largest
|
||||
run scripts/bench_downloader.py --urls-file scripts/bench-urls/apple_sustained_4x585mb.txt --impersonate Chrome131
|
||||
run scripts/bench_downloader.py --urls-file scripts/bench-urls/akamai_4k_segments.txt --impersonate Chrome131
|
||||
fi
|
||||
|
||||
echo; echo "report saved: $OUT"
|
||||
Reference in New Issue
Block a user