diff --git a/tests/remote/unit/test_remote_client.py b/tests/remote/unit/test_remote_client.py
index 2461d13a..80e8fa4d 100644
--- a/tests/remote/unit/test_remote_client.py
+++ b/tests/remote/unit/test_remote_client.py
@@ -85,6 +85,42 @@ def test_4xx_raises_systemexit_with_logged_error(client: RemoteClient, caplog: p
assert "no such session" in caplog.text
+PROXY_502 = "
unshackle.dev | 502: Bad gateway"
+
+
+@responses.activate
+def test_proxy_502_page_is_retried_once(client: RemoteClient, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("unshackle.core.remote_service.time.sleep", lambda _s: None)
+ url = "http://srv:8786/api/session/abc/license"
+ responses.add(responses.POST, url, body=PROXY_502, status=502, content_type="text/html")
+ responses.add(responses.POST, url, json={"keys": {}}, status=200)
+ assert client.post("/api/session/abc/license", {}) == {"keys": {}}
+ assert len(responses.calls) == 2
+
+
+@responses.activate
+def test_second_proxy_502_page_exits(client: RemoteClient, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("unshackle.core.remote_service.time.sleep", lambda _s: None)
+ url = "http://srv:8786/api/session/abc/license"
+ responses.add(responses.POST, url, body=PROXY_502, status=502, content_type="text/html")
+ with pytest.raises(SystemExit):
+ client.post("/api/session/abc/license", {})
+ assert len(responses.calls) == 2
+
+
+@responses.activate
+def test_server_json_503_is_not_retried(client: RemoteClient) -> None:
+ responses.add(
+ responses.GET,
+ "http://srv:8786/api/services",
+ json={"error_code": "SERVICE_UNAVAILABLE", "message": "busy"},
+ status=503,
+ )
+ with pytest.raises(SystemExit):
+ client.get("/api/services")
+ assert len(responses.calls) == 1
+
+
@responses.activate
def test_401_falls_back_to_next_auth_header(client: RemoteClient) -> None:
seen: list[tuple[str | None, str | None]] = []
diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py
index 8a0bb36e..226792e9 100644
--- a/unshackle/commands/dl.py
+++ b/unshackle/commands/dl.py
@@ -4877,6 +4877,8 @@ class dl:
return not drm.content_keys or bool(track_kid and track_kid not in drm.content_keys)
if missing_track_key():
+ from unshackle.core.remote_service import RemoteError
+
try:
licence(
drm_system="playready" if drm.__class__.__name__ == "PlayReady" else "widevine",
@@ -4884,7 +4886,7 @@ class dl:
)
except click.ClickException:
raise
- except Exception as e:
+ except (Exception, RemoteError) as e:
self.log.debug(f"Server CDM licence with an empty challenge failed: {e!r}")
server_vault_keys = {} if cdm_only else getattr(svc_for_cdm, "server_vault_keys", {})
diff --git a/unshackle/core/remote_service.py b/unshackle/core/remote_service.py
index 202d7b52..f8bca0fa 100644
--- a/unshackle/core/remote_service.py
+++ b/unshackle/core/remote_service.py
@@ -144,6 +144,7 @@ class RemoteClient:
payload: Dict[str, Any] = (
{"json": data} if body is None else {"data": body, "headers": {"Content-Type": "application/octet-stream"}}
)
+ gateway_retried = False
while True:
try:
with self.in_flight(endpoint):
@@ -160,6 +161,16 @@ class RemoteClient:
# servers differ on which header carries the key, so retry the rest before giving up
if resp.status_code == 401 and self.next_auth_header():
continue
+ # a proxy HTML page, not the server's JSON, means a dropped tunnel stream, which one retry gets past
+ if (
+ resp.status_code in (502, 503, 504)
+ and not gateway_retried
+ and "json" not in resp.headers.get("Content-Type", "")
+ ):
+ gateway_retried = True
+ log.debug(f"HTTP {resp.status_code} from the proxy in front of the server on {endpoint}, retrying once")
+ time.sleep(1)
+ continue
break
if resp.status_code < 400 and "octet-stream" in resp.headers.get("Content-Type", ""):
return {"body": resp.content}
@@ -1870,7 +1881,7 @@ class RemoteService:
for kid_hex, key_hex in keys.items():
drm_obj.content_keys[UUID(hex=kid_hex)] = key_hex
return challenge
- except Exception as e:
+ except (Exception, RemoteError) as e:
self.log.warning("server_cdm license failed: %s", e)
else:
self.log.warning(f"Track {track.id} has no PSSH to send to the server CDM")