perf(manifests): decode manifests without charset detection

Parse an MPD from bytes and read every M3U8 playlist as UTF-8 unless the server sent a charset. Fixes mojibake on text/* responses.
This commit is contained in:
imSp4rky
2026-10-13 16:11:13 -06:00
parent 4fbaf2a61a
commit e7bb825def
10 changed files with 114 additions and 48 deletions
+3 -5
View File
@@ -31,6 +31,7 @@ from unshackle.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_C
from unshackle.core.providers.anilist import parse_anilist_ref
from unshackle.core.proxies.resolve import initialize_proxy_providers, resolve_proxy
from unshackle.core.services import Services
from unshackle.core.session import utf8_text
from unshackle.core.titles import Episode, Movie, Song, Title_T
from unshackle.core.tracks import Audio, Subtitle, Tracks, Video
from unshackle.core.utilities import declared_kwargs, kind_only_conflict
@@ -4327,7 +4328,7 @@ def ensure_track_drm(track: Any, session: Any = None, init_data: Optional[bytes]
from unshackle.core.manifests import HLS
http = getattr(track, "session", None) or session
playlist = m3u8.loads(http.get(track.url).text, uri=track.url) if http else m3u8.load(track.url)
playlist = m3u8.loads(utf8_text(http.get(track.url)), uri=track.url) if http else m3u8.load(track.url)
keys = [k for k in (playlist.keys or []) + (playlist.session_keys or []) if k is not None]
for key in keys:
try:
@@ -4807,14 +4808,11 @@ async def session_segment_filter_handler(
def unwanted_segments() -> list[str]:
import m3u8
import requests
response = session.service_instance.session.get(track.url)
if not getattr(response, "ok", True):
raise ValueError(f"playlist request failed: {response.status_code}")
if isinstance(response, requests.Response):
response.encoding = response.encoding or "utf-8"
playlist = m3u8.loads(response.text, uri=str(track.url))
playlist = m3u8.loads(utf8_text(response), uri=str(track.url))
return [segment.absolute_uri for segment in playlist.segments if segment_filter(segment)]
try:
+2 -1
View File
@@ -25,6 +25,7 @@ from unshackle.core.config import config
from unshackle.core.console import console
from unshackle.core.constants import AnyTrack
from unshackle.core.drm import key_args as drm_key_args
from unshackle.core.session import utf8_text
from unshackle.core.utilities import get_boxes, log_event
from unshackle.core.utils.subprocess import ffprobe
@@ -180,7 +181,7 @@ class PlayReady:
if track.descriptor == track.Descriptor.HLS:
m3u_url = track.url
master = m3u8.loads(session.get(m3u_url).text, uri=m3u_url)
master = m3u8.loads(utf8_text(session.get(m3u_url)), uri=m3u_url)
pssh_boxes.extend(
Box.parse(base64.b64decode(x.uri.split(",")[-1]))
for x in (master.session_keys or master.keys)
+2 -1
View File
@@ -22,6 +22,7 @@ from unshackle.core.config import config
from unshackle.core.console import console
from unshackle.core.constants import AnyTrack
from unshackle.core.drm import key_args as drm_key_args
from unshackle.core.session import utf8_text
from unshackle.core.utilities import get_boxes, log_event
from unshackle.core.utils.subprocess import ffprobe
@@ -94,7 +95,7 @@ class Widevine:
if track.descriptor == track.Descriptor.HLS:
m3u_url = track.url
master = m3u8.loads(session.get(m3u_url).text, uri=m3u_url)
master = m3u8.loads(utf8_text(session.get(m3u_url)), uri=m3u_url)
pssh_boxes.extend(
Box.parse(base64.b64decode(x.uri.split(",")[-1]))
for x in (master.session_keys or master.keys)
+2 -1
View File
@@ -25,6 +25,7 @@ from unshackle.core.remote_service import (
resolve_proxy_arg,
segment_uri_filter,
)
from unshackle.core.session import utf8_text
from unshackle.core.titles import Episode, Movies, Series, Title_T, Titles_T, remap_titles
from unshackle.core.tracks import Audio, Chapter, Chapters, Subtitle, Tracks, Video
from unshackle.core.tracks.attachment import Attachment
@@ -69,7 +70,7 @@ def hls_playlist_kids(track: Any, session: Optional[requests.Session] = None) ->
"""
http = session or requests.Session()
try:
playlist = m3u8.loads(http.get(str(track.url)).text, uri=str(track.url))
playlist = m3u8.loads(utf8_text(http.get(str(track.url))), uri=str(track.url))
keys = [k for k in playlist.keys if k is not None and k.method != "NONE"]
except Exception as e:
log.warning(f"Cannot read the media playlist of {track.id} for its KIDs: {e!r}")
+9 -4
View File
@@ -19,7 +19,7 @@ from zlib import crc32
import requests
from langcodes import Language, tag_is_valid
from lxml.etree import Element, ElementTree
from lxml.etree import Element, ElementTree, XMLSyntaxError
from pyplayready.system.pssh import PSSH as PR_PSSH
from pywidevine.cdm import Cdm as WidevineCdm
from pywidevine.pssh import PSSH
@@ -118,6 +118,7 @@ class DASH:
@classmethod
def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **args: Any) -> DASH:
"""Fetch and parse an MPD. The XML declaration gives the encoding, then the HTTP charset."""
if not url:
raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.")
if not isinstance(url, str):
@@ -138,11 +139,15 @@ class DASH:
log_event(
"manifest_dash_fetch",
level="DEBUG",
message=f"Fetched DASH manifest ({len(res.text)} bytes)",
context={"url": safe_display_url(url), "size": len(res.text)},
message=f"Fetched DASH manifest ({len(res.content)} bytes)",
context={"url": safe_display_url(url), "size": len(res.content)},
)
return DASH.from_text(res.text, url)
try:
return cls(load_xml(res.content), url)
except XMLSyntaxError:
# no declaration and not UTF-8: only the HTTP charset can name the encoding
return cls.from_text(res.text, url)
@classmethod
def from_text(cls, text: str, url: str) -> DASH:
+12 -25
View File
@@ -37,7 +37,7 @@ from unshackle.core.drm.segment_decrypt import SegmentDecrypter, can_use
from unshackle.core.drm.verify import decrypt_track
from unshackle.core.events import events
from unshackle.core.manifests.dash import RollingMerge
from unshackle.core.session import RnetResponse, RnetSession
from unshackle.core.session import RnetResponse, RnetSession, utf8_text
from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Tracks, Video, resume
from unshackle.core.tracks.track import DRM_PREFERENCE_TYPES, assert_fragments_decrypted
from unshackle.core.utilities import get_extension, is_close_match, log_event, try_ensure_utf8
@@ -81,17 +81,11 @@ class HLS:
res = session.get(url, **args)
if isinstance(res, requests.Response):
if not res.ok:
raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res)
res.encoding = res.encoding or "utf-8"
content = res.text
elif isinstance(res, RnetResponse):
if not res.ok:
raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res)
content = res.text
else:
if not isinstance(res, (requests.Response, RnetResponse)):
raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(res)}")
if not res.ok:
raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res)
content = utf8_text(res)
master = m3u8.loads(content, uri=url)
@@ -397,9 +391,7 @@ class HLS:
session = Session()
res = session.get(variant_url)
if isinstance(res, requests.Response):
res.encoding = res.encoding or "utf-8"
variant = m3u8.loads(res.text, uri=variant_url)
variant = m3u8.loads(utf8_text(res), uri=variant_url)
if not variant.segments:
return None
@@ -659,17 +651,13 @@ class HLS:
master = m3u8.load(str(track.from_file))
else:
response = session.get(track.url)
if isinstance(response, requests.Response) or isinstance(response, RnetResponse):
if not response.ok:
log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}")
sys.exit(1)
if isinstance(response, requests.Response):
response.encoding = response.encoding or "utf-8"
playlist_text = response.text
else:
if not isinstance(response, (requests.Response, RnetResponse)):
raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(response)}")
if not response.ok:
log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}")
sys.exit(1)
master = m3u8.loads(playlist_text, uri=track.url)
master = m3u8.loads(utf8_text(response), uri=track.url)
if not master.segments:
log.error("Track's HLS playlist has no segments, expecting an invariant M3U8 playlist.")
@@ -1385,8 +1373,7 @@ class HLS:
if not value and data.uri:
if not session:
session = Session()
res = session.get(urljoin(manifest.base_uri or "", data.uri))
value = res.text
value = utf8_text(session.get(urljoin(manifest.base_uri or "", data.uri)))
if not value:
continue
+13 -1
View File
@@ -15,7 +15,7 @@ from typing import Any, Optional
from urllib.parse import urlencode, urlparse, urlunparse
import rnet
from requests import HTTPError, Request
from requests import HTTPError, Request, Response
from requests.structures import CaseInsensitiveDict
from unshackle.core.config import config
@@ -227,6 +227,18 @@ class RnetResponse:
pass
def utf8_text(res: Response | RnetResponse) -> str:
"""Return the response text, decoded as UTF-8 unless the server sent a charset.
With no charset, requests assumes ISO-8859-1 for a `text/*` type and runs charset detection
on the full body for any other type. Use this only for a format that is UTF-8 by
specification, such as M3U8.
"""
if isinstance(res, Response) and "charset" not in res.headers.get("content-type", "").lower():
res.encoding = "utf-8"
return res.text
class RnetSessionHeaders(CaseInsensitiveDict):
"""Dict-like headers that write to the rnet client through update()."""
+3 -5
View File
@@ -20,7 +20,7 @@ import click
from construct import ConstructError
from langcodes import Language
from pymp4.parser import Box
from requests import Response, Session
from requests import Session
from requests.adapters import HTTPAdapter, Retry
from unshackle.core import binaries
@@ -31,7 +31,7 @@ from unshackle.core.downloaders import requests
from unshackle.core.drm import DRM_T, ClearKeyCENC, PlayReady, Widevine
from unshackle.core.drm.verify import decrypt_track
from unshackle.core.events import events
from unshackle.core.session import RnetSession
from unshackle.core.session import RnetSession, utf8_text
from unshackle.core.tracks import resume
from unshackle.core.utilities import get_boxes, log_event, try_ensure_utf8
from unshackle.core.utils.subprocess import ffprobe
@@ -1141,9 +1141,7 @@ class Track:
session = getattr(self, "session", None) or Session()
response = session.get(self.url)
if isinstance(response, Response):
response.encoding = response.encoding or "utf-8"
playlist = m3u8.loads(response.text, self.url)
playlist = m3u8.loads(utf8_text(response), self.url)
drm_list = []
+2 -2
View File
@@ -13,7 +13,7 @@ from urllib.parse import urljoin
from requests import Session
from unshackle.core.binaries import FFProbe
from unshackle.core.session import RnetSession
from unshackle.core.session import RnetSession, utf8_text
from unshackle.core.utils.subprocess import log_tool_run
if TYPE_CHECKING:
@@ -320,7 +320,7 @@ def extract_hls(track: "Track", session: Union[Session, RnetSession]) -> list[Se
playlist_url = track.url if isinstance(track.url, str) else track.url[0]
res = session.get(playlist_url, timeout=PROBE_TIMEOUT)
playlist = m3u8.loads(res.text, uri=playlist_url)
playlist = m3u8.loads(utf8_text(res), uri=playlist_url)
out: list[Segment] = []
for segment in playlist.segments: