mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-12 12:05:07 +00:00
fix(vaults): back off longer on 429 and shorten the skip messages
This commit is contained in:
@@ -11,9 +11,15 @@ from unshackle.core import __version__
|
||||
from unshackle.core.session import POOL_MAX_SIZE
|
||||
|
||||
|
||||
RATE_LIMIT_RETRIES = 3
|
||||
|
||||
|
||||
class RateLimited(Exception):
|
||||
"""The vault still answered 429 after the retries."""
|
||||
|
||||
def __init__(self, message: str = f"still rate limited after {RATE_LIMIT_RETRIES} retries"):
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
UNREACHABLE = (RequestsConnectionError, Timeout, ConnectionError, TimeoutError)
|
||||
|
||||
@@ -25,11 +31,11 @@ def vault_session(timeout: float) -> Session:
|
||||
failure is not retried: a lookup waits on every remote vault, so a retry multiplies the timeout.
|
||||
"""
|
||||
retry = Retry(
|
||||
total=3,
|
||||
total=RATE_LIMIT_RETRIES,
|
||||
connect=0,
|
||||
read=0,
|
||||
other=0,
|
||||
backoff_factor=0.5,
|
||||
backoff_factor=1,
|
||||
status_forcelist=[429],
|
||||
allowed_methods=None,
|
||||
retry_after_max=int(timeout),
|
||||
|
||||
@@ -248,7 +248,7 @@ class Vaults:
|
||||
timeout, not one per KID."""
|
||||
if isinstance(error, (RateLimited, *UNREACHABLE)) and vault not in self.unreachable:
|
||||
self.unreachable.add(vault)
|
||||
log.warning(f"Vault '{vault.name}' is unavailable, so this run does not use it again")
|
||||
log.warning(f"Skipping Vault '{vault.name}' for the rest of this run")
|
||||
|
||||
|
||||
__all__ = ("Vaults",)
|
||||
|
||||
@@ -38,7 +38,7 @@ class API(Vault):
|
||||
allow_redirects=False,
|
||||
)
|
||||
if response.status_code == 429:
|
||||
raise Exceptions.TooManyRequests(f"Vault {self.name} rate limited the lookup")
|
||||
raise Exceptions.TooManyRequests()
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
@@ -172,7 +172,7 @@ class API(Vault):
|
||||
continue
|
||||
|
||||
if response.status_code == 429:
|
||||
raise Exceptions.TooManyRequests(f"Vault {self.name} rate limited the push")
|
||||
raise Exceptions.TooManyRequests()
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except (ValueError, HTTPError):
|
||||
|
||||
@@ -89,7 +89,7 @@ class HTTP(Vault):
|
||||
return {"status": "not_found"}
|
||||
|
||||
if r.status_code == 429:
|
||||
raise RateLimited(f"Vault {self.name} rate limited the request")
|
||||
raise RateLimited()
|
||||
|
||||
if not r.ok:
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user