diff --git a/scripts/bench_suite.bat b/scripts/bench_suite.bat new file mode 100644 index 00000000..9b6fad81 --- /dev/null +++ b/scripts/bench_suite.bat @@ -0,0 +1,12 @@ +@echo off +REM Run the downloader benchmark suite and save a readable report for comparison. +REM Offline synthetic + fault runs by default; --live also hits public test CDNs +REM (needs network + ffprobe on PATH). +REM scripts\bench_suite.bat (offline suite) +REM scripts\bench_suite.bat --live (+ live/network tests) +REM Output: scripts\bench-results\_.txt +setlocal +cd /d "%~dp0.." +if not exist "scripts\bench-results" mkdir "scripts\bench-results" +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0bench_suite.ps1" %* +endlocal diff --git a/scripts/bench_suite.ps1 b/scripts/bench_suite.ps1 new file mode 100644 index 00000000..de4141a9 --- /dev/null +++ b/scripts/bench_suite.ps1 @@ -0,0 +1,106 @@ +# Run the downloader benchmark suite and save a readable report for comparison. +# Offline synthetic + fault runs by default; -live also hits public test CDNs +# (needs network + ffprobe on PATH). Usually launched via bench_suite.bat. +# powershell -File scripts\bench_suite.ps1 # offline +# powershell -File scripts\bench_suite.ps1 --live # + live/network tests +param([string]$mode = "offline") +$ErrorActionPreference = "Continue" +Set-Location (Join-Path $PSScriptRoot "..") + +$live = ($mode -eq "--live" -or $mode -eq "-live" -or $mode -eq "live") +$stamp = Get-Date -Format "yyyyMMdd-HHmmss" +$out = "scripts\bench-results\$($env:COMPUTERNAME)_$stamp.txt" +if (-not (Test-Path "scripts\bench-results")) { New-Item -ItemType Directory -Path "scripts\bench-results" | Out-Null } + +function Run($a) { + Write-Output "" + Write-Output ("==================== python " + ($a -join " ") + " ====================") + # stringify the merged stream: PS 5.1 wraps native stderr lines in ErrorRecords, + # which render as NativeCommandError blocks (expected-fail runs looked like crashes) + & uv run python @a 2>&1 | ForEach-Object { "$_" } + if ($LASTEXITCODE -ne 0) { + Write-Output ("(exit code: " + $LASTEXITCODE + ")") + } +} + +# collect everything, then tee to the report so tables land in the file too +& { + Write-Output "# unshackle downloader benchmark" + Write-Output ("host: " + $env:COMPUTERNAME) + Write-Output ("date: " + (Get-Date -Format "yyyy-MM-dd HH:mm:ss")) + Write-Output ("git: " + (git rev-parse --abbrev-ref HEAD) + " @ " + (git rev-parse --short HEAD)) + $cpu = Get-CimInstance Win32_Processor | Select-Object -First 1 + Write-Output ("cpu: " + $cpu.Name + " (" + $cpu.NumberOfCores + "c/" + $cpu.NumberOfLogicalProcessors + "t)") + $mem = [math]::Round((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB) + $os = Get-CimInstance Win32_OperatingSystem + Write-Output ("mem: " + $mem + " GiB") + Write-Output ("os: " + $os.Caption + " " + $os.Version) + Write-Output ("python: " + (& uv run python -V 2>&1 | ForEach-Object { "$_" })) + Write-Output ("mode: " + $(if ($live) { "live" } else { "offline" })) + + # negotiated link speed per NIC; shows the physical ceiling before the throughput run + Write-Output "" + Write-Output "==================== nic link speed ====================" + Get-NetAdapter | Where-Object Status -eq 'Up' | + ForEach-Object { Write-Output ("{0}: {1} ({2})" -f $_.Name, $_.LinkSpeed, $_.InterfaceDescription) } + + # link baseline: max line speed, so downloader numbers can be read against it + # (e.g. "2Gbe -> ~1900 Mbps expected ceiling"). Uses whatever CLI exists. + Write-Output "" + Write-Output "==================== speedtest baseline ====================" + if (Get-Command speedtest -ErrorAction SilentlyContinue) { + & speedtest --accept-license --accept-gdpr 2>&1 | ForEach-Object { "$_" } + } elseif (Get-Command npx -ErrorAction SilentlyContinue) { + # Netflix fast.com; download + upload + latency + server, clean when piped + & npx --yes fast-cli --upload --verbose 2>&1 | ForEach-Object { "$_" } + } elseif (Get-Command speedtest-cli -ErrorAction SilentlyContinue) { + & speedtest-cli --simple 2>&1 | ForEach-Object { "$_" } + } else { + Write-Output "no speedtest CLI found (install Ookla 'speedtest', have npx, or 'pip install speedtest-cli') - skipping" + } + + # 1. Synthetic downloader benchmark (offline, deterministic) + Run @("scripts/bench_downloader.py", "--segments", "64", "--workers", "1,2,4,8,16", "--runs", "3") + Run @("scripts/bench_downloader.py", "--segments", "64", "--workers", "1,2,4,8,16", "--adaptive") + Run @("scripts/bench_downloader.py", "--workers", "4,8,16", "--no-read1") + + # 2. Fault paths (--fault-stall is EXPECTED to fail + exit non-zero) + Run @("scripts/bench_downloader.py", "--fault-503", "2", "--fast-timeouts", "--segments", "8") + Run @("scripts/bench_downloader.py", "--fault-reset", "2", "--fast-timeouts", "--segments", "8") + Run @("scripts/bench_downloader.py", "--fault-stall", "1", "--fast-timeouts", "--segments", "8") + + # 3. Multiprocess path (>=24 segments engages spawned children; strided chunk split). + # big batch: at small sizes the fixed spawn cost drowns the throughput signal + Run @("scripts/bench_downloader.py", "--segments", "128", "--workers", "8", "--procs", "2") + Run @("scripts/bench_downloader.py", "--segments", "128", "--workers", "8", "--procs", "4") + Run @("scripts/bench_downloader.py", "--fault-stall", "1", "--fast-timeouts", "--segments", "32", "--workers", "8", "--procs", "2") + + # 4. Hedge + slow tail (last segments crawl; hedging and striding bound the tail) + Run @("scripts/bench_downloader.py", "--fault-tail-slow", "4", "--fast-timeouts", "--segments", "32", "--workers", "8") + Run @("scripts/bench_downloader.py", "--fault-tail-slow", "4", "--fast-timeouts", "--segments", "32", "--workers", "8", "--procs", "4") + Run @("scripts/bench_downloader.py", "--fault-tail-slow", "2", "--fast-timeouts", "--adaptive", "--segments", "16", "--workers", "8") + + # 5. rnet session path (loopback through the TLS-fingerprint client's own read loop) + Run @("scripts/bench_downloader.py", "--segments", "32", "--workers", "8", "--impersonate", "Chrome131") + + # 6. Leak regression tests (locked handles / leaked workers; unlink checks bite on Windows) + Write-Output "" + Write-Output "==================== pytest tests/test_downloader_leaks.py ====================" + & uv run pytest tests/test_downloader_leaks.py -q 2>&1 | ForEach-Object { "$_" } + if ($LASTEXITCODE -ne 0) { + Write-Output "(pytest failed or missing - run 'uv sync' to install dev deps)" + } + + # 7. Live network tests (opt-in) + if ($live) { + Run @("scripts/live_manifest_test.py", "--workers", "16", "--track", "largest") + Run @("scripts/live_segmentbase_ab.py", "--track", "largest") + Run @("scripts/bench_downloader.py", "--urls-file", "scripts/bench-urls/apple_sustained_4x585mb.txt", "--impersonate", "Chrome131") + Run @("scripts/bench_downloader.py", "--urls-file", "scripts/bench-urls/akamai_4k_segments.txt", "--impersonate", "Chrome131") + } + + Write-Output "" + Write-Output ("report saved: " + $out) +} | Tee-Object -Variable report +# utf8 (not Tee-Object's utf16) so reports diff and grep cleanly across machines +$report | Out-File -FilePath $out -Encoding utf8 diff --git a/scripts/bench_suite.sh b/scripts/bench_suite.sh new file mode 100644 index 00000000..45f13821 --- /dev/null +++ b/scripts/bench_suite.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Run the downloader benchmark suite and save a readable report for comparison. +# Offline synthetic + fault runs by default; --live also hits public test CDNs +# (needs network + ffprobe on PATH). +# ./scripts/bench_suite.sh # offline suite +# ./scripts/bench_suite.sh --live # + live/network tests +# Output: scripts/bench-results/_.txt +set -uo pipefail +cd "$(dirname "$0")/.." + +OUT_DIR="scripts/bench-results" +mkdir -p "$OUT_DIR" +STAMP="$(date -u +%Y%m%d-%H%M%S)" +OUT="$OUT_DIR/$(hostname)_${STAMP}.txt" + +# tee everything (stdout+stderr) to the report so tables land in the file too +exec > >(tee "$OUT") 2>&1 + +echo "# unshackle downloader benchmark" +echo "host: $(hostname)" +echo "date: $(date -u '+%Y-%m-%d %H:%M:%S') UTC" +echo "git: $(git rev-parse --abbrev-ref HEAD) @ $(git rev-parse --short HEAD)" +echo "cpu: $(grep -m1 'model name' /proc/cpuinfo | cut -d: -f2- | sed 's/^ *//') ($(nproc) threads)" +echo "mem: $(free -h --si 2>/dev/null | awk '/^Mem:/{print $2}')" +echo "os: $(uname -sr)" +echo "python: $(uv run python -V 2>&1)" +echo "mode: ${1:-offline}" + +run() { echo; echo "==================== $* ===================="; uv run python "$@"; } + +# link baseline: establishes the max line speed so downloader numbers can be read +# against it (e.g. "2Gbe -> ~1900 Mbps expected ceiling"). Uses whatever CLI exists. +# negotiated link speed per NIC (from sysfs); shows the physical ceiling, e.g. 2500 = 2.5GbE +nic_link_speed() { + echo; echo "==================== nic link speed ====================" + local found=0 + for d in /sys/class/net/*; do + local n state spd + n=$(basename "$d") + [ "$n" = "lo" ] && continue + state=$(cat "$d/operstate" 2>/dev/null) + [ "$state" = "up" ] || continue + spd=$(cat "$d/speed" 2>/dev/null) + echo "$n: ${spd:-?} Mbps" + found=1 + done + [ "$found" = 1 ] || echo "no up interfaces with readable speed (virtual/WSL/wifi report -1)" +} +nic_link_speed + +speedtest_baseline() { + echo; echo "==================== speedtest baseline ====================" + if command -v speedtest >/dev/null 2>&1; then + speedtest --accept-license --accept-gdpr 2>&1 || echo "speedtest failed" + elif command -v npx >/dev/null 2>&1; then + # Netflix fast.com; download + upload + latency + server, clean when piped + npx --yes fast-cli --upload --verbose 2>&1 || echo "npx fast-cli failed" + elif command -v speedtest-cli >/dev/null 2>&1; then + speedtest-cli --simple 2>&1 || echo "speedtest-cli failed" + else + echo "no speedtest CLI found (install Ookla 'speedtest', have npx, or 'pipx install speedtest-cli') - skipping" + fi +} +speedtest_baseline + +# 1. Synthetic downloader benchmark (offline, deterministic) +run scripts/bench_downloader.py --segments 64 --workers 1,2,4,8,16 --runs 3 +run scripts/bench_downloader.py --segments 64 --workers 1,2,4,8,16 --adaptive +run scripts/bench_downloader.py --workers 4,8,16 --no-read1 + +# 2. Fault paths (fast timeouts; --fault-stall is EXPECTED to fail + exit non-zero) +run scripts/bench_downloader.py --fault-503 2 --fast-timeouts --segments 8 +run scripts/bench_downloader.py --fault-reset 2 --fast-timeouts --segments 8 +run scripts/bench_downloader.py --fault-stall 1 --fast-timeouts --segments 8 || true + +# 3. Multiprocess path (>=24 segments engages spawned children; strided chunk split). +# big batch: at small sizes the fixed spawn cost drowns the throughput signal +run scripts/bench_downloader.py --segments 128 --workers 8 --procs 2 +run scripts/bench_downloader.py --segments 128 --workers 8 --procs 4 +run scripts/bench_downloader.py --fault-stall 1 --fast-timeouts --segments 32 --workers 8 --procs 2 || true + +# 4. Hedge + slow tail (last segments crawl; hedging and striding bound the tail) +run scripts/bench_downloader.py --fault-tail-slow 4 --fast-timeouts --segments 32 --workers 8 +run scripts/bench_downloader.py --fault-tail-slow 4 --fast-timeouts --segments 32 --workers 8 --procs 4 +run scripts/bench_downloader.py --fault-tail-slow 2 --fast-timeouts --adaptive --segments 16 --workers 8 + +# 5. rnet session path (loopback through the TLS-fingerprint client's own read loop) +run scripts/bench_downloader.py --segments 32 --workers 8 --impersonate Chrome131 || true + +# 6. Leak regression tests (locked handles / leaked workers; unlink checks bite on Windows) +echo; echo "==================== pytest tests/test_downloader_leaks.py ====================" +uv run pytest tests/test_downloader_leaks.py -q || echo "(pytest failed or missing - run 'uv sync' to install dev deps)" + +# 7. Live network tests (opt-in) +if [[ "${1:-}" == "--live" ]]; then + run scripts/live_manifest_test.py --workers 16 --track largest + run scripts/live_segmentbase_ab.py --track largest + run scripts/bench_downloader.py --urls-file scripts/bench-urls/apple_sustained_4x585mb.txt --impersonate Chrome131 + run scripts/bench_downloader.py --urls-file scripts/bench-urls/akamai_4k_segments.txt --impersonate Chrome131 +fi + +echo; echo "report saved: $OUT"