fix(downloader): fail closed on rnet content codings outside the old allowlist

The rnet branch zeroed content_length only for gzip/deflate/br, so an
unknown coding (zstd, future tokens) kept a bogus length and, worse, the
drop-the-partial-instead-of-Range-resuming guard could never fire on the
rnet path because content_encoded stayed False. Use the same fail-closed
_is_content_encoded check the requests branch uses.
This commit is contained in:
imSp4rky
2026-08-07 11:37:54 -06:00
parent 90a6164c56
commit e45cf15dd7
2 changed files with 50 additions and 5 deletions
@@ -1,4 +1,5 @@
import gzip
import importlib
import socket
import threading
@@ -108,3 +109,48 @@ def test_encoded_body_discards_a_partial_instead_of_range_resuming(tmp_path, enc
save_path.with_name("sub.vtt.!dev").write_bytes(PLAIN[:5000])
list(download(url=encoding_server.url, save_path=save_path, session=rq.Session()))
assert save_path.read_bytes() == PLAIN
class _FakeRnetResponse:
def __init__(self, status_code, headers, body):
self.status_code = status_code
self.headers = headers
self.content_length = len(body)
self._body = body
def raise_for_status(self):
pass
def stream(self):
yield self._body
def close(self):
pass
class _FakeRnetSession:
"""Duck-typed rnet stand-in: an encoded 206 for a Range request (the resume the
fail-closed guard must reject), an encoded 200 for the clean restart."""
def __init__(self, body):
self.body = body
self.range_requests = 0
def get(self, url, stream=True, **kwargs):
headers = kwargs.get("headers") or {}
if any(k.lower() == "range" for k in headers):
self.range_requests += 1
return _FakeRnetResponse(206, {"Content-Encoding": "zstd"}, b"\x00mid-stream-slice\x00")
return _FakeRnetResponse(200, {"Content-Encoding": "zstd"}, self.body)
def test_rnet_encoded_body_discards_a_partial_instead_of_range_resuming(tmp_path, monkeypatch):
dl = importlib.import_module("unshackle.core.downloaders.requests")
monkeypatch.setattr(dl, "_is_rnet_session", lambda s: isinstance(s, _FakeRnetSession))
session = _FakeRnetSession(PLAIN)
save_path = tmp_path / "sub.vtt"
save_path.with_name("sub.vtt.!dev").write_bytes(PLAIN[:5000])
list(download(url="http://127.0.0.1:1/sub.vtt", save_path=save_path, session=session))
# the encoded 206 must be dropped (never appended); the restart fetches the whole body
assert session.range_requests == 1
assert save_path.read_bytes() == PLAIN
+4 -5
View File
@@ -714,12 +714,11 @@ def download(
# server ignored the slice's Range and sent the whole parent; writing that
# as the segment would silently corrupt the merge, so fail the attempt
raise IOError(f"expected 206 for byte-range segment, got {stream.status_code}")
content_encoded = False
if use_rnet:
content_length = stream.content_length or 0
ce = (stream.headers.get("Content-Encoding") or stream.headers.get("content-encoding") or "").lower()
if ce in ("gzip", "deflate", "br"):
content_length = 0
content_encoded = _is_content_encoded(
stream.headers.get("Content-Encoding") or stream.headers.get("content-encoding")
)
content_length = 0 if content_encoded else (stream.content_length or 0)
else:
content_encoded = _is_content_encoded(stream.headers.get("Content-Encoding"))
try: