mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-06-15 21:47:24 +00:00
Compare commits
7 Commits
7dd6323be5
...
feat/cross
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1a636d3db5 | ||
|
|
e0dbd0b046 | ||
|
|
e02aa66843 | ||
|
|
c82bb5fe34 | ||
|
|
ec2ecfe7b4 | ||
|
|
15acaea208 | ||
|
|
def18a4c44 |
@@ -25,6 +25,7 @@ import click
|
||||
import jsonpickle
|
||||
import yaml
|
||||
from construct import ConstError
|
||||
from langcodes import Language
|
||||
from pymediainfo import MediaInfo
|
||||
from pyplayready.cdm import Cdm as PlayReadyCdm
|
||||
from pyplayready.device import Device as PlayReadyDevice
|
||||
@@ -63,7 +64,7 @@ from unshackle.core.tracks.hybrid import Hybrid
|
||||
from unshackle.core.utilities import (find_font_with_fallbacks, get_debug_logger, get_system_fonts, init_debug_logger,
|
||||
is_close_match, suggest_font_packages, time_elapsed_since)
|
||||
from unshackle.core.utils import tags
|
||||
from unshackle.core.utils.click_types import (AUDIO_CODEC_LIST, LANGUAGE_RANGE, QUALITY_LIST, SEASON_RANGE,
|
||||
from unshackle.core.utils.click_types import (AUDIO_CODEC_LIST, LANGUAGE_RANGE, OFFSET, QUALITY_LIST, SEASON_RANGE,
|
||||
ContextData, MultipleChoice, MultipleVideoCodecChoice,
|
||||
SubtitleCodecChoice)
|
||||
from unshackle.core.utils.collections import merge_dict
|
||||
@@ -505,6 +506,12 @@ class dl:
|
||||
@click.option(
|
||||
"--reset-cache", "reset_cache", is_flag=True, default=False, help="Clear title cache before fetching."
|
||||
)
|
||||
@click.option(
|
||||
"--worst",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Select the lowest bitrate track within the specified quality. Requires -q/--quality.",
|
||||
)
|
||||
@click.option(
|
||||
"--best-available",
|
||||
"best_available",
|
||||
@@ -512,6 +519,58 @@ class dl:
|
||||
default=False,
|
||||
help="Continue with best available quality if requested resolutions are not available.",
|
||||
)
|
||||
@click.option(
|
||||
"--cross-video",
|
||||
nargs=2,
|
||||
type=(str, str),
|
||||
default=None,
|
||||
help="Cross-mux: use video from another service. Format: SERVICE URL.",
|
||||
)
|
||||
@click.option(
|
||||
"--cross-audio",
|
||||
nargs=2,
|
||||
type=(str, str),
|
||||
default=None,
|
||||
help="Cross-mux: use audio from another service. Format: SERVICE URL.",
|
||||
)
|
||||
@click.option(
|
||||
"--cross-subtitles",
|
||||
nargs=2,
|
||||
type=(str, str),
|
||||
default=None,
|
||||
help="Cross-mux: use subtitles from another service. Format: SERVICE URL.",
|
||||
)
|
||||
@click.option(
|
||||
"--cross-chapters",
|
||||
nargs=2,
|
||||
type=(str, str),
|
||||
default=None,
|
||||
help="Cross-mux: use chapters from another service. Format: SERVICE URL.",
|
||||
)
|
||||
@click.option(
|
||||
"--cross-audio-offset",
|
||||
type=OFFSET,
|
||||
default=None,
|
||||
help="Timing offset for cross-sourced audio, e.g. '10s', '500ms', '-5.5s'.",
|
||||
)
|
||||
@click.option(
|
||||
"--cross-subtitle-offset",
|
||||
type=OFFSET,
|
||||
default=None,
|
||||
help="Timing offset for cross-sourced subtitles, e.g. '10s', '500ms', '-5.5s'.",
|
||||
)
|
||||
@click.option(
|
||||
"--cross-profile",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Profile to use for cross-service credentials. Defaults to --profile.",
|
||||
)
|
||||
@click.option(
|
||||
"--cross-wanted",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Override episode mapping for cross-services, e.g. 'S01E02'.",
|
||||
)
|
||||
@click.pass_context
|
||||
def cli(ctx: click.Context, **kwargs: Any) -> dl:
|
||||
return dl(ctx, **kwargs)
|
||||
@@ -531,6 +590,14 @@ class dl:
|
||||
animeapi_id: Optional[str] = None,
|
||||
enrich: bool = False,
|
||||
output_dir: Optional[Path] = None,
|
||||
cross_video: Optional[tuple[str, str]] = None,
|
||||
cross_audio: Optional[tuple[str, str]] = None,
|
||||
cross_subtitles: Optional[tuple[str, str]] = None,
|
||||
cross_chapters: Optional[tuple[str, str]] = None,
|
||||
cross_audio_offset: Optional[int] = None,
|
||||
cross_subtitle_offset: Optional[int] = None,
|
||||
cross_profile: Optional[str] = None,
|
||||
cross_wanted: Optional[str] = None,
|
||||
*_: Any,
|
||||
**__: Any,
|
||||
):
|
||||
@@ -579,6 +646,16 @@ class dl:
|
||||
self.animeapi_title: Optional[str] = None
|
||||
self.output_dir = output_dir
|
||||
|
||||
# Cross-mux settings
|
||||
self.cross_video = cross_video
|
||||
self.cross_audio = cross_audio
|
||||
self.cross_subtitles = cross_subtitles
|
||||
self.cross_chapters = cross_chapters
|
||||
self.cross_audio_offset = cross_audio_offset
|
||||
self.cross_subtitle_offset = cross_subtitle_offset
|
||||
self.cross_profile = cross_profile or profile
|
||||
self.cross_wanted = cross_wanted
|
||||
|
||||
if animeapi_id:
|
||||
from unshackle.core.utils.animeapi import resolve_animeapi
|
||||
|
||||
@@ -947,6 +1024,218 @@ class dl:
|
||||
# able to keep `self` as the first positional
|
||||
self.cli._result_callback = self.result
|
||||
|
||||
def _instantiate_cross_service(self, tag: str, url: str) -> tuple[Service, str]:
|
||||
"""
|
||||
Instantiate a cross-service for cross-mux by tag and URL.
|
||||
|
||||
Returns (service_instance, title_url) after authentication.
|
||||
"""
|
||||
tag = Services.get_tag(tag)
|
||||
service_cls = Services.load(tag)
|
||||
|
||||
# Build service config for the cross-service
|
||||
service_config_path = Services.get_path(tag) / config.filenames.config
|
||||
if service_config_path.exists():
|
||||
cross_service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8"))
|
||||
else:
|
||||
cross_service_config = {}
|
||||
|
||||
# Load CDM for the cross-service
|
||||
cross_cdm = self.get_cdm(tag, self.cross_profile)
|
||||
|
||||
# Build a synthetic Click context for the cross-service
|
||||
cross_ctx_obj = ContextData(
|
||||
config=cross_service_config,
|
||||
cdm=cross_cdm,
|
||||
proxy_providers=self.proxy_providers,
|
||||
profile=self.cross_profile,
|
||||
)
|
||||
|
||||
# Extract the title argument from the URL using TITLE_RE
|
||||
title_id = url
|
||||
if hasattr(service_cls, "TITLE_RE"):
|
||||
m = re.match(service_cls.TITLE_RE, url)
|
||||
if m:
|
||||
# Try named group 'title_id' first, then 'id', then group(1)
|
||||
title_id = m.group("title_id") if "title_id" in m.groupdict() else (
|
||||
m.group("id") if "id" in m.groupdict() else m.group(1)
|
||||
)
|
||||
|
||||
# Build kwargs from the service cli command's params with defaults
|
||||
cli_cmd = service_cls.cli
|
||||
kwargs = {"title": title_id}
|
||||
for param in cli_cmd.params:
|
||||
if param.name and param.name != "title" and param.name not in kwargs:
|
||||
kwargs[param.name] = param.default
|
||||
|
||||
# Create a parent context that mimics what dl.__init__ sets up.
|
||||
# Services access ctx.parent.params for various dl-level options,
|
||||
# so we provide a complete set of defaults to avoid KeyError.
|
||||
parent_ctx = click.Context(self.cli, info_name="dl")
|
||||
parent_ctx.params = {
|
||||
"no_proxy": True,
|
||||
"proxy": None,
|
||||
"proxy_query": None,
|
||||
"proxy_provider": None,
|
||||
"vcodec": [],
|
||||
"acodec": [],
|
||||
"range_": [Video.Range.SDR],
|
||||
"best_available": False,
|
||||
"profile": self.cross_profile,
|
||||
"quality": [],
|
||||
"wanted": None,
|
||||
"video_only": False,
|
||||
"audio_only": False,
|
||||
"subs_only": False,
|
||||
"chapters_only": False,
|
||||
"list_": False,
|
||||
"skip_dl": False,
|
||||
"no_cache": False,
|
||||
"reset_cache": False,
|
||||
}
|
||||
ctx = click.Context(cli_cmd, parent=parent_ctx, info_name=tag)
|
||||
ctx.obj = cross_ctx_obj
|
||||
|
||||
# Instantiate the service
|
||||
cross_service = service_cls(ctx, **kwargs)
|
||||
|
||||
# Authenticate
|
||||
cookies = self.get_cookie_jar(tag, self.cross_profile)
|
||||
credential = self.get_credentials(tag, self.cross_profile)
|
||||
cross_service.authenticate(cookies, credential)
|
||||
|
||||
return cross_service
|
||||
|
||||
def _match_cross_title(
|
||||
self, primary_title: Title_T, cross_titles: Any
|
||||
) -> Optional[Title_T]:
|
||||
"""Match a primary title to its counterpart in cross-service titles."""
|
||||
if isinstance(primary_title, Movie):
|
||||
# For movies, the cross URL should resolve to the same movie
|
||||
if hasattr(cross_titles, "__iter__"):
|
||||
for t in cross_titles:
|
||||
if isinstance(t, Movie):
|
||||
return t
|
||||
return None
|
||||
|
||||
if isinstance(primary_title, Episode):
|
||||
if self.cross_wanted:
|
||||
# Manual override: parse S01E02 format
|
||||
m = re.match(r"S(\d+)E(\d+)", self.cross_wanted, re.IGNORECASE)
|
||||
if m:
|
||||
wanted_season = int(m.group(1))
|
||||
wanted_episode = int(m.group(2))
|
||||
for t in cross_titles:
|
||||
if isinstance(t, Episode) and t.season == wanted_season and t.number == wanted_episode:
|
||||
return t
|
||||
self.log.warning(
|
||||
f"Cross-wanted S{wanted_season:02d}E{wanted_episode:02d} not found in cross-service"
|
||||
)
|
||||
return None
|
||||
|
||||
# Auto-match by season + episode number
|
||||
for t in cross_titles:
|
||||
if isinstance(t, Episode) and t.season == primary_title.season and t.number == primary_title.number:
|
||||
return t
|
||||
|
||||
self.log.warning(
|
||||
f"No cross-service match for S{primary_title.season:02d}E{primary_title.number:02d}"
|
||||
)
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
def _process_cross_services(self, title: Title_T) -> dict[str, tuple[Service, Title_T, Tracks]]:
|
||||
"""
|
||||
Process all cross-service specs and return fetched tracks per track type.
|
||||
|
||||
Returns dict like {"video": (service, matched_title, tracks), ...}
|
||||
"""
|
||||
cross_specs: list[tuple[str, Optional[tuple[str, str]]]] = [
|
||||
("video", self.cross_video),
|
||||
("audio", self.cross_audio),
|
||||
("subtitles", self.cross_subtitles),
|
||||
("chapters", self.cross_chapters),
|
||||
]
|
||||
|
||||
# Cache instantiated services by (tag, url) to avoid duplicate auth
|
||||
service_cache: dict[tuple[str, str], Service] = {}
|
||||
result: dict[str, tuple[Service, Title_T, Tracks]] = {}
|
||||
|
||||
for track_type, spec in cross_specs:
|
||||
if not spec:
|
||||
continue
|
||||
|
||||
tag, url = spec
|
||||
cache_key = (Services.get_tag(tag), url)
|
||||
|
||||
if cache_key not in service_cache:
|
||||
self.log.info(f"Cross-mux: loading {track_type} from {tag}")
|
||||
cross_service = self._instantiate_cross_service(tag, url)
|
||||
service_cache[cache_key] = cross_service
|
||||
else:
|
||||
cross_service = service_cache[cache_key]
|
||||
|
||||
# Get titles from cross-service
|
||||
cross_titles = cross_service.get_titles()
|
||||
|
||||
# Match the primary title to a cross-service title
|
||||
cross_title = self._match_cross_title(title, cross_titles)
|
||||
if not cross_title:
|
||||
self.log.warning(f"Cross-mux: could not match title for {track_type} from {tag}, skipping")
|
||||
continue
|
||||
|
||||
# Get tracks from cross-service
|
||||
cross_tracks = cross_service.get_tracks(cross_title)
|
||||
cross_chapters = cross_service.get_chapters(cross_title)
|
||||
cross_tracks.chapters = cross_chapters
|
||||
|
||||
result[track_type] = (cross_service, cross_title, cross_tracks)
|
||||
|
||||
return result
|
||||
|
||||
def _apply_cross_tracks(
|
||||
self,
|
||||
title: Title_T,
|
||||
cross_results: dict[str, tuple[Service, Title_T, Tracks]],
|
||||
) -> None:
|
||||
"""Replace primary tracks with cross-service tracks and mark with metadata."""
|
||||
if "video" in cross_results:
|
||||
cross_service, cross_title, cross_tracks = cross_results["video"]
|
||||
title.tracks.videos = cross_tracks.videos
|
||||
for track in title.tracks.videos:
|
||||
track.data["_cross_service"] = cross_service
|
||||
track.data["_cross_title"] = cross_title
|
||||
track.data["cross_source"] = cross_service.__class__.__name__
|
||||
|
||||
if "audio" in cross_results:
|
||||
cross_service, cross_title, cross_tracks = cross_results["audio"]
|
||||
title.tracks.audio = cross_tracks.audio
|
||||
for track in title.tracks.audio:
|
||||
track.data["_cross_service"] = cross_service
|
||||
track.data["_cross_title"] = cross_title
|
||||
track.data["cross_source"] = cross_service.__class__.__name__
|
||||
if self.cross_audio_offset:
|
||||
track.data["cross_offset_ms"] = self.cross_audio_offset
|
||||
|
||||
if "subtitles" in cross_results:
|
||||
cross_service, cross_title, cross_tracks = cross_results["subtitles"]
|
||||
title.tracks.subtitles = cross_tracks.subtitles
|
||||
for track in title.tracks.subtitles:
|
||||
track.data["_cross_service"] = cross_service
|
||||
track.data["_cross_title"] = cross_title
|
||||
track.data["cross_source"] = cross_service.__class__.__name__
|
||||
if self.cross_subtitle_offset:
|
||||
track.data["cross_offset_ms"] = self.cross_subtitle_offset
|
||||
|
||||
if "chapters" in cross_results:
|
||||
_, _, cross_tracks = cross_results["chapters"]
|
||||
title.tracks.chapters = cross_tracks.chapters
|
||||
|
||||
@property
|
||||
def has_cross_mux(self) -> bool:
|
||||
return any([self.cross_video, self.cross_audio, self.cross_subtitles, self.cross_chapters])
|
||||
|
||||
def result(
|
||||
self,
|
||||
service: Service,
|
||||
@@ -990,6 +1279,7 @@ class dl:
|
||||
no_mux: bool,
|
||||
workers: Optional[int],
|
||||
downloads: int,
|
||||
worst: bool,
|
||||
best_available: bool,
|
||||
split_audio: Optional[bool] = None,
|
||||
*_: Any,
|
||||
@@ -1015,6 +1305,10 @@ class dl:
|
||||
self.log.error("--require-subs and --s-lang cannot be used together")
|
||||
sys.exit(1)
|
||||
|
||||
if worst and not quality:
|
||||
self.log.error("--worst requires -q/--quality to be specified")
|
||||
sys.exit(1)
|
||||
|
||||
if select_titles and wanted:
|
||||
self.log.error("--select-titles and -w/--wanted cannot be used together")
|
||||
sys.exit(1)
|
||||
@@ -1404,6 +1698,25 @@ class dl:
|
||||
level="INFO", operation="get_tracks", service=self.service, context=tracks_info
|
||||
)
|
||||
|
||||
# Cross-mux: replace tracks from cross-services if configured
|
||||
if self.has_cross_mux:
|
||||
with console.status("Cross-mux: fetching tracks from cross-services...", spinner="dots"):
|
||||
try:
|
||||
cross_results = self._process_cross_services(title)
|
||||
if cross_results:
|
||||
self._apply_cross_tracks(title, cross_results)
|
||||
cross_sources = ", ".join(
|
||||
f"{k}={v[0].__class__.__name__}" for k, v in cross_results.items()
|
||||
)
|
||||
self.log.info(f"Cross-mux: applied tracks from {cross_sources}")
|
||||
except Exception as e:
|
||||
self.log.error(f"Cross-mux failed: {e}")
|
||||
if self.debug_logger:
|
||||
self.debug_logger.log_error(
|
||||
"cross_mux", e, service=self.service, context={"title": str(title)}
|
||||
)
|
||||
raise
|
||||
|
||||
# strip SDH subs to non-SDH if no equivalent same-lang non-SDH is available
|
||||
# uses a loose check, e.g, wont strip en-US SDH sub if a non-SDH en-GB is available
|
||||
# Check if automatic SDH stripping is enabled in config
|
||||
@@ -1608,20 +1921,18 @@ class dl:
|
||||
for resolution, color_range, codec in product(
|
||||
quality or [None], non_hybrid_ranges, vcodec or [None]
|
||||
):
|
||||
match = next(
|
||||
(
|
||||
t
|
||||
for t in non_hybrid_tracks
|
||||
if (
|
||||
not resolution
|
||||
or t.height == resolution
|
||||
or int(t.width * (9 / 16)) == resolution
|
||||
)
|
||||
and (not color_range or t.range == color_range)
|
||||
and (not codec or t.codec == codec)
|
||||
),
|
||||
None,
|
||||
)
|
||||
candidates = [
|
||||
t
|
||||
for t in non_hybrid_tracks
|
||||
if (
|
||||
not resolution
|
||||
or t.height == resolution
|
||||
or int(t.width * (9 / 16)) == resolution
|
||||
)
|
||||
and (not color_range or t.range == color_range)
|
||||
and (not codec or t.codec == codec)
|
||||
]
|
||||
match = candidates[-1] if worst and candidates else next(iter(candidates), None)
|
||||
if match and match not in non_hybrid_selected:
|
||||
non_hybrid_selected.append(match)
|
||||
|
||||
@@ -1631,20 +1942,18 @@ class dl:
|
||||
for resolution, color_range, codec in product(
|
||||
quality or [None], range_ or [None], vcodec or [None]
|
||||
):
|
||||
match = next(
|
||||
(
|
||||
t
|
||||
for t in title.tracks.videos
|
||||
if (
|
||||
not resolution
|
||||
or t.height == resolution
|
||||
or int(t.width * (9 / 16)) == resolution
|
||||
)
|
||||
and (not color_range or t.range == color_range)
|
||||
and (not codec or t.codec == codec)
|
||||
),
|
||||
None,
|
||||
)
|
||||
candidates = [
|
||||
t
|
||||
for t in title.tracks.videos
|
||||
if (
|
||||
not resolution
|
||||
or t.height == resolution
|
||||
or int(t.width * (9 / 16)) == resolution
|
||||
)
|
||||
and (not color_range or t.range == color_range)
|
||||
and (not codec or t.codec == codec)
|
||||
]
|
||||
match = candidates[-1] if worst and candidates else next(iter(candidates), None)
|
||||
if match and match not in selected_videos:
|
||||
selected_videos.append(match)
|
||||
title.tracks.videos = selected_videos
|
||||
@@ -1942,21 +2251,35 @@ class dl:
|
||||
(
|
||||
pool.submit(
|
||||
track.download,
|
||||
session=service.session,
|
||||
session=(
|
||||
track.data["_cross_service"].session
|
||||
if track.data.get("_cross_service")
|
||||
else service.session
|
||||
),
|
||||
prepare_drm=partial(
|
||||
partial(self.prepare_drm, table=download_table),
|
||||
track=track,
|
||||
title=title,
|
||||
title=track.data.get("_cross_title", title),
|
||||
certificate=partial(
|
||||
service.get_widevine_service_certificate,
|
||||
title=title,
|
||||
(
|
||||
track.data["_cross_service"].get_widevine_service_certificate
|
||||
if track.data.get("_cross_service")
|
||||
else service.get_widevine_service_certificate
|
||||
),
|
||||
title=track.data.get("_cross_title", title),
|
||||
track=track,
|
||||
),
|
||||
licence=partial(
|
||||
service.get_playready_license
|
||||
if is_playready_cdm(self.cdm)
|
||||
else service.get_widevine_license,
|
||||
title=title,
|
||||
(
|
||||
track.data["_cross_service"].get_playready_license
|
||||
if track.data.get("_cross_service") and is_playready_cdm(self.cdm)
|
||||
else track.data["_cross_service"].get_widevine_license
|
||||
if track.data.get("_cross_service")
|
||||
else service.get_playready_license
|
||||
if is_playready_cdm(self.cdm)
|
||||
else service.get_widevine_license
|
||||
),
|
||||
title=track.data.get("_cross_title", title),
|
||||
track=track,
|
||||
),
|
||||
cdm_only=cdm_only,
|
||||
@@ -2025,49 +2348,6 @@ class dl:
|
||||
dl_time = time_elapsed_since(dl_start_time)
|
||||
console.print(Padding(f"Track downloads finished in [progress.elapsed]{dl_time}[/]", (0, 5)))
|
||||
|
||||
video_track_n = 0
|
||||
|
||||
while (
|
||||
not title.tracks.subtitles
|
||||
and not no_subs
|
||||
and not (hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES)
|
||||
and not video_only
|
||||
and not no_video
|
||||
and len(title.tracks.videos) > video_track_n
|
||||
and any(
|
||||
x.get("codec_name", "").startswith("eia_")
|
||||
for x in ffprobe(title.tracks.videos[video_track_n].path).get("streams", [])
|
||||
)
|
||||
):
|
||||
with console.status(f"Checking Video track {video_track_n + 1} for Closed Captions..."):
|
||||
try:
|
||||
# TODO: Figure out the real language, it might be different
|
||||
# EIA-CC tracks sadly don't carry language information :(
|
||||
# TODO: Figure out if the CC language is original lang or not.
|
||||
# Will need to figure out above first to do so.
|
||||
video_track = title.tracks.videos[video_track_n]
|
||||
track_id = f"ccextractor-{video_track.id}"
|
||||
cc_lang = title.language or video_track.language
|
||||
cc = video_track.ccextractor(
|
||||
track_id=track_id,
|
||||
out_path=config.directories.temp
|
||||
/ config.filenames.subtitle.format(id=track_id, language=cc_lang),
|
||||
language=cc_lang,
|
||||
original=False,
|
||||
)
|
||||
if cc:
|
||||
# will not appear in track listings as it's added after all times it lists
|
||||
title.tracks.add(cc)
|
||||
self.log.info(f"Extracted a Closed Caption from Video track {video_track_n + 1}")
|
||||
else:
|
||||
self.log.info(f"No Closed Captions were found in Video track {video_track_n + 1}")
|
||||
except EnvironmentError:
|
||||
self.log.error(
|
||||
"Cannot extract Closed Captions as the ccextractor executable was not found..."
|
||||
)
|
||||
break
|
||||
video_track_n += 1
|
||||
|
||||
# Subtitle output mode configuration (for sidecar originals)
|
||||
subtitle_output_mode = config.subtitle.get("output_mode", "mux")
|
||||
sidecar_format = config.subtitle.get("sidecar_format", "srt")
|
||||
@@ -2133,6 +2413,57 @@ class dl:
|
||||
if has_decrypted:
|
||||
self.log.info(f"Decrypted tracks with {decrypt_tool}")
|
||||
|
||||
# Extract Closed Captions from decrypted video tracks
|
||||
if (
|
||||
not no_subs
|
||||
and not (hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES)
|
||||
and not video_only
|
||||
and not no_video
|
||||
):
|
||||
for video_track_n, video_track in enumerate(title.tracks.videos):
|
||||
has_manifest_cc = bool(getattr(video_track, "closed_captions", None))
|
||||
has_eia_cc = (
|
||||
not has_manifest_cc
|
||||
and not title.tracks.subtitles
|
||||
and any(
|
||||
x.get("codec_name", "").startswith("eia_")
|
||||
for x in ffprobe(video_track.path).get("streams", [])
|
||||
)
|
||||
)
|
||||
if not has_manifest_cc and not has_eia_cc:
|
||||
continue
|
||||
|
||||
with console.status(f"Checking Video track {video_track_n + 1} for Closed Captions..."):
|
||||
try:
|
||||
cc_lang = (
|
||||
Language.get(video_track.closed_captions[0]["language"])
|
||||
if has_manifest_cc and video_track.closed_captions[0].get("language")
|
||||
else title.language or video_track.language
|
||||
)
|
||||
track_id = f"ccextractor-{video_track.id}"
|
||||
cc = video_track.ccextractor(
|
||||
track_id=track_id,
|
||||
out_path=config.directories.temp
|
||||
/ config.filenames.subtitle.format(id=track_id, language=cc_lang),
|
||||
language=cc_lang,
|
||||
original=False,
|
||||
)
|
||||
if cc:
|
||||
cc.cc = True
|
||||
title.tracks.add(cc)
|
||||
self.log.info(
|
||||
f"Extracted a Closed Caption from Video track {video_track_n + 1}"
|
||||
)
|
||||
else:
|
||||
self.log.info(
|
||||
f"No Closed Captions were found in Video track {video_track_n + 1}"
|
||||
)
|
||||
except EnvironmentError:
|
||||
self.log.error(
|
||||
"Cannot extract Closed Captions as the ccextractor executable was not found..."
|
||||
)
|
||||
break
|
||||
|
||||
# Now repack the decrypted tracks
|
||||
with console.status("Repackaging tracks with FFMPEG..."):
|
||||
has_repacked = False
|
||||
|
||||
@@ -431,14 +431,24 @@ def download(
|
||||
raise ValueError(error)
|
||||
|
||||
# Yield aggregate progress for this call's downloads
|
||||
if total_size > 0:
|
||||
# Yield both advance (bytes downloaded this iteration) and total for rich progress
|
||||
if dl_speed != -1:
|
||||
yield dict(downloaded=f"{filesize.decimal(dl_speed)}/s", advance=0, completed=total_completed, total=total_size)
|
||||
progress_data = {"advance": 0}
|
||||
|
||||
if len(gids) > 1:
|
||||
# Multi-file mode (e.g., HLS): Return the count of completed segments
|
||||
progress_data["completed"] = len(completed)
|
||||
progress_data["total"] = len(gids)
|
||||
else:
|
||||
# Single-file mode: Return the total bytes downloaded
|
||||
progress_data["completed"] = total_completed
|
||||
if total_size > 0:
|
||||
progress_data["total"] = total_size
|
||||
else:
|
||||
yield dict(advance=0, completed=total_completed, total=total_size)
|
||||
elif dl_speed != -1:
|
||||
yield dict(downloaded=f"{filesize.decimal(dl_speed)}/s")
|
||||
progress_data["total"] = None
|
||||
|
||||
if dl_speed != -1:
|
||||
progress_data["downloaded"] = f"{filesize.decimal(dl_speed)}/s"
|
||||
|
||||
yield progress_data
|
||||
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
|
||||
@@ -356,6 +356,19 @@ class PlayReady:
|
||||
key_hex = key if isinstance(key, str) else key.hex()
|
||||
key_args.extend(["--key", f"{kid_hex}:{key_hex}"])
|
||||
|
||||
# Some services use a blank/zero default KID in the tenc box,
|
||||
# but the real KID for the license server. Add zero-KID fallback entries so
|
||||
# mp4decrypt can match when the file's default KID is all zeros.
|
||||
zero_kid = "00" * 16
|
||||
existing_kids = {
|
||||
kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "")
|
||||
for kid in self.content_keys
|
||||
}
|
||||
if zero_kid not in existing_kids:
|
||||
for key in self.content_keys.values():
|
||||
key_hex = key if isinstance(key, str) else key.hex()
|
||||
key_args.extend(["--key", f"{zero_kid}:{key_hex}"])
|
||||
|
||||
cmd = [
|
||||
str(binaries.Mp4decrypt),
|
||||
"--show-progress",
|
||||
|
||||
@@ -276,6 +276,19 @@ class Widevine:
|
||||
key_hex = key if isinstance(key, str) else key.hex()
|
||||
key_args.extend(["--key", f"{kid_hex}:{key_hex}"])
|
||||
|
||||
# Some services use a blank/zero default KID in the tenc box,
|
||||
# but the real KID for the license server. Add zero-KID fallback entries so
|
||||
# mp4decrypt can match when the file's default KID is all zeros.
|
||||
zero_kid = "00" * 16
|
||||
existing_kids = {
|
||||
kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "")
|
||||
for kid in self.content_keys
|
||||
}
|
||||
if zero_kid not in existing_kids:
|
||||
for key in self.content_keys.values():
|
||||
key_hex = key if isinstance(key, str) else key.hex()
|
||||
key_args.extend(["--key", f"{zero_kid}:{key_hex}"])
|
||||
|
||||
cmd = [
|
||||
str(binaries.Mp4decrypt),
|
||||
"--show-progress",
|
||||
|
||||
@@ -112,6 +112,15 @@ class HLS:
|
||||
session_drm = HLS.get_all_drm(session_keys)
|
||||
|
||||
audio_codecs_by_group_id: dict[str, Audio.Codec] = {}
|
||||
cc_by_group_id: dict[str, list[dict[str, Any]]] = {}
|
||||
for media in self.manifest.media:
|
||||
if media.type == "CLOSED-CAPTIONS":
|
||||
cc_by_group_id.setdefault(media.group_id, []).append({
|
||||
"language": media.language,
|
||||
"name": media.name,
|
||||
"instream_id": media.instream_id,
|
||||
"characteristics": media.characteristics,
|
||||
})
|
||||
tracks = Tracks()
|
||||
|
||||
for playlist in self.manifest.playlists:
|
||||
@@ -161,6 +170,9 @@ class HLS:
|
||||
width=playlist.stream_info.resolution[0] if playlist.stream_info.resolution else None,
|
||||
height=playlist.stream_info.resolution[1] if playlist.stream_info.resolution else None,
|
||||
fps=playlist.stream_info.frame_rate,
|
||||
closed_captions=cc_by_group_id.get(
|
||||
(playlist.stream_info.closed_captions or "").strip('"'), []
|
||||
),
|
||||
)
|
||||
if primary_track_type is Video
|
||||
else {}
|
||||
@@ -471,6 +483,8 @@ class HLS:
|
||||
final_save_path = HLS._finalize_n_m3u8dl_re_output(track=track, save_dir=save_dir, save_path=save_path)
|
||||
progress(downloaded="Downloaded")
|
||||
track.path = final_save_path
|
||||
if session_drm:
|
||||
track.drm = None
|
||||
events.emit(events.Types.TRACK_DOWNLOADED, track=track)
|
||||
return
|
||||
|
||||
@@ -775,6 +789,10 @@ class HLS:
|
||||
progress(downloaded="Downloaded")
|
||||
|
||||
track.path = save_path
|
||||
|
||||
if session_drm:
|
||||
track.drm = None
|
||||
|
||||
events.emit(events.Types.TRACK_DOWNLOADED, track=track)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -145,7 +145,14 @@ class ISM:
|
||||
fragment_time += duration_frag
|
||||
|
||||
track_id = hashlib.md5(
|
||||
f"{codec}-{track_lang}-{ql.get('Bitrate') or 0}-{ql.get('Index') or 0}".encode()
|
||||
"{codec}-{lang}-{bitrate}-{index}-{name}-{url}".format(
|
||||
codec=codec,
|
||||
lang=track_lang,
|
||||
bitrate=ql.get("Bitrate") or 0,
|
||||
index=ql.get("Index") or 0,
|
||||
name=stream_index.get("Name") or "",
|
||||
url=stream_index.get("Url") or "",
|
||||
).encode()
|
||||
).hexdigest()
|
||||
|
||||
data = {
|
||||
|
||||
@@ -44,6 +44,17 @@ FINGERPRINT_PRESETS = {
|
||||
"akamai": "4:16777216|16711681|0|m,p,a,s",
|
||||
"description": "OkHttp 5.x (BoringSSL TLS stack)",
|
||||
},
|
||||
"shield_okhttp": {
|
||||
"ja3": (
|
||||
"771," # TLS 1.2
|
||||
"4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53," # Ciphers (OkHttp 4.11)
|
||||
"0-23-65281-10-11-35-16-5-13-51-45-43-21," # Extensions (incl padding ext 21)
|
||||
"29-23-24," # Named groups (x25519, secp256r1, secp384r1)
|
||||
"0" # EC point formats
|
||||
),
|
||||
"akamai": "4:16777216|16711681|0|m,p,a,s",
|
||||
"description": "NVIDIA SHIELD Android TV OkHttp 4.11 (captured JA3)",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -103,53 +103,78 @@ class Tracks:
|
||||
tree = Tree("", hide_root=True)
|
||||
for track_type in self.TRACK_ORDER_MAP:
|
||||
tracks = list(x for x in all_tracks if isinstance(x, track_type))
|
||||
if not tracks:
|
||||
continue
|
||||
num_tracks = len(tracks)
|
||||
track_type_plural = track_type.__name__ + ("s" if track_type != Audio and num_tracks != 1 else "")
|
||||
tracks_tree = tree.add(f"[repr.number]{num_tracks}[/] {track_type_plural}")
|
||||
for track in tracks:
|
||||
if add_progress and track_type not in (Chapter, Attachment):
|
||||
progress = Progress(
|
||||
SpinnerColumn(finished_text=""),
|
||||
BarColumn(),
|
||||
"•",
|
||||
TimeRemainingColumn(compact=True, elapsed_when_finished=True),
|
||||
"•",
|
||||
TextColumn("[progress.data.speed]{task.fields[downloaded]}"),
|
||||
console=console,
|
||||
speed_estimate_period=10,
|
||||
if tracks:
|
||||
num_tracks = len(tracks)
|
||||
track_type_plural = track_type.__name__ + ("s" if track_type != Audio and num_tracks != 1 else "")
|
||||
tracks_tree = tree.add(f"[repr.number]{num_tracks}[/] {track_type_plural}")
|
||||
for track in tracks:
|
||||
if add_progress and track_type not in (Chapter, Attachment):
|
||||
progress = Progress(
|
||||
SpinnerColumn(finished_text=""),
|
||||
BarColumn(),
|
||||
"•",
|
||||
TimeRemainingColumn(compact=True, elapsed_when_finished=True),
|
||||
"•",
|
||||
TextColumn("[progress.data.speed]{task.fields[downloaded]}"),
|
||||
console=console,
|
||||
speed_estimate_period=10,
|
||||
)
|
||||
task = progress.add_task("", downloaded="-")
|
||||
state = {"total": 100.0}
|
||||
|
||||
def update_track_progress(
|
||||
task_id: int = task,
|
||||
_state: dict[str, float] = state,
|
||||
_progress: Progress = progress,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""
|
||||
Ensure terminal status states render as a fully completed bar.
|
||||
|
||||
Some downloaders can report completed slightly below total
|
||||
before emitting the final "Downloaded" state.
|
||||
"""
|
||||
if "total" in kwargs and kwargs["total"] is not None:
|
||||
_state["total"] = kwargs["total"]
|
||||
|
||||
downloaded_state = kwargs.get("downloaded")
|
||||
if downloaded_state in {"Downloaded", "Decrypted", "[yellow]SKIPPED"}:
|
||||
kwargs["completed"] = _state["total"]
|
||||
_progress.update(task_id=task_id, **kwargs)
|
||||
|
||||
progress_callables.append(update_track_progress)
|
||||
track_table = Table.grid()
|
||||
track_table.add_row(str(track)[6:], style="text2")
|
||||
track_table.add_row(progress)
|
||||
tracks_tree.add(track_table)
|
||||
else:
|
||||
tracks_tree.add(str(track)[6:], style="text2")
|
||||
|
||||
# Show Closed Captions right after Subtitles (even if no subtitle tracks exist)
|
||||
if track_type is Subtitle:
|
||||
seen_cc: set[str] = set()
|
||||
unique_cc: list[str] = []
|
||||
for video in (x for x in all_tracks if isinstance(x, Video)):
|
||||
for cc in getattr(video, "closed_captions", []):
|
||||
lang = cc.get("language", "und")
|
||||
name = cc.get("name", "")
|
||||
instream_id = cc.get("instream_id", "")
|
||||
key = f"{lang}|{instream_id}"
|
||||
if key in seen_cc:
|
||||
continue
|
||||
seen_cc.add(key)
|
||||
parts = [f"[CC] | {lang}"]
|
||||
if name:
|
||||
parts.append(name)
|
||||
if instream_id:
|
||||
parts.append(instream_id)
|
||||
unique_cc.append(" | ".join(parts))
|
||||
if unique_cc:
|
||||
cc_tree = tree.add(
|
||||
f"[repr.number]{len(unique_cc)}[/] Closed Caption{'s' if len(unique_cc) != 1 else ''}"
|
||||
)
|
||||
task = progress.add_task("", downloaded="-")
|
||||
state = {"total": 100.0}
|
||||
|
||||
def update_track_progress(
|
||||
task_id: int = task,
|
||||
_state: dict[str, float] = state,
|
||||
_progress: Progress = progress,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""
|
||||
Ensure terminal status states render as a fully completed bar.
|
||||
|
||||
Some downloaders can report completed slightly below total
|
||||
before emitting the final "Downloaded" state.
|
||||
"""
|
||||
if "total" in kwargs and kwargs["total"] is not None:
|
||||
_state["total"] = kwargs["total"]
|
||||
|
||||
downloaded_state = kwargs.get("downloaded")
|
||||
if downloaded_state in {"Downloaded", "Decrypted", "[yellow]SKIPPED"}:
|
||||
kwargs["completed"] = _state["total"]
|
||||
_progress.update(task_id=task_id, **kwargs)
|
||||
|
||||
progress_callables.append(update_track_progress)
|
||||
track_table = Table.grid()
|
||||
track_table.add_row(str(track)[6:], style="text2")
|
||||
track_table.add_row(progress)
|
||||
tracks_tree.add(track_table)
|
||||
else:
|
||||
tracks_tree.add(str(track)[6:], style="text2")
|
||||
for cc_str in unique_cc:
|
||||
cc_tree.add(cc_str, style="text2")
|
||||
|
||||
return tree, progress_callables
|
||||
|
||||
@@ -452,25 +477,25 @@ class Tracks:
|
||||
if not at.path or not at.path.exists():
|
||||
raise ValueError("Audio Track must be downloaded before muxing...")
|
||||
events.emit(events.Types.TRACK_MULTIPLEX, track=at)
|
||||
cl.extend(
|
||||
[
|
||||
"--track-name",
|
||||
f"0:{at.get_track_name() or ''}",
|
||||
"--language",
|
||||
f"0:{at.language}",
|
||||
"--default-track",
|
||||
f"0:{at.is_original_lang}",
|
||||
"--visual-impaired-flag",
|
||||
f"0:{at.descriptive}",
|
||||
"--original-flag",
|
||||
f"0:{at.is_original_lang}",
|
||||
"--compression",
|
||||
"0:none", # disable extra compression
|
||||
"(",
|
||||
str(at.path),
|
||||
")",
|
||||
]
|
||||
)
|
||||
audio_args = [
|
||||
"--track-name",
|
||||
f"0:{at.get_track_name() or ''}",
|
||||
"--language",
|
||||
f"0:{at.language}",
|
||||
"--default-track",
|
||||
f"0:{at.is_original_lang}",
|
||||
"--visual-impaired-flag",
|
||||
f"0:{at.descriptive}",
|
||||
"--original-flag",
|
||||
f"0:{at.is_original_lang}",
|
||||
"--compression",
|
||||
"0:none", # disable extra compression
|
||||
]
|
||||
|
||||
if at.data.get("cross_offset_ms"):
|
||||
audio_args.extend(["--sync", f"0:{at.data['cross_offset_ms']}"])
|
||||
|
||||
cl.extend(audio_args + ["(", str(at.path), ")"])
|
||||
|
||||
if not skip_subtitles:
|
||||
for st in self.subtitles:
|
||||
@@ -478,29 +503,29 @@ class Tracks:
|
||||
raise ValueError("Text Track must be downloaded before muxing...")
|
||||
events.emit(events.Types.TRACK_MULTIPLEX, track=st)
|
||||
default = bool(self.audio and is_close_match(st.language, [self.audio[0].language]) and st.forced)
|
||||
cl.extend(
|
||||
[
|
||||
"--track-name",
|
||||
f"0:{st.get_track_name() or ''}",
|
||||
"--language",
|
||||
f"0:{st.language}",
|
||||
"--sub-charset",
|
||||
"0:UTF-8",
|
||||
"--forced-track",
|
||||
f"0:{st.forced}",
|
||||
"--default-track",
|
||||
f"0:{default}",
|
||||
"--hearing-impaired-flag",
|
||||
f"0:{st.sdh}",
|
||||
"--original-flag",
|
||||
f"0:{st.is_original_lang}",
|
||||
"--compression",
|
||||
"0:none", # disable extra compression (probably zlib)
|
||||
"(",
|
||||
str(st.path),
|
||||
")",
|
||||
]
|
||||
)
|
||||
sub_args = [
|
||||
"--track-name",
|
||||
f"0:{st.get_track_name() or ''}",
|
||||
"--language",
|
||||
f"0:{st.language}",
|
||||
"--sub-charset",
|
||||
"0:UTF-8",
|
||||
"--forced-track",
|
||||
f"0:{st.forced}",
|
||||
"--default-track",
|
||||
f"0:{default}",
|
||||
"--hearing-impaired-flag",
|
||||
f"0:{st.sdh}",
|
||||
"--original-flag",
|
||||
f"0:{st.is_original_lang}",
|
||||
"--compression",
|
||||
"0:none", # disable extra compression (probably zlib)
|
||||
]
|
||||
|
||||
if st.data.get("cross_offset_ms"):
|
||||
sub_args.extend(["--sync", f"0:{st.data['cross_offset_ms']}"])
|
||||
|
||||
cl.extend(sub_args + ["(", str(st.path), ")"])
|
||||
|
||||
if self.chapters:
|
||||
chapters_path = config.directories.temp / config.filenames.chapters.format(
|
||||
|
||||
@@ -200,6 +200,7 @@ class Video(Track):
|
||||
height: Optional[int] = None,
|
||||
fps: Optional[Union[str, int, float]] = None,
|
||||
scan_type: Optional[Video.ScanType] = None,
|
||||
closed_captions: Optional[list[dict[str, Any]]] = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -264,6 +265,7 @@ class Video(Track):
|
||||
raise ValueError("Expected fps to be a number, float, or a string as numerator/denominator form, " + str(e))
|
||||
|
||||
self.scan_type = scan_type
|
||||
self.closed_captions: list[dict[str, Any]] = closed_captions or []
|
||||
self.needs_duration_fix = False
|
||||
|
||||
def __str__(self) -> str:
|
||||
@@ -346,22 +348,27 @@ class Video(Track):
|
||||
if not binaries.CCExtractor:
|
||||
raise EnvironmentError("ccextractor executable was not found.")
|
||||
|
||||
# ccextractor often fails in weird ways unless we repack
|
||||
self.repackage()
|
||||
|
||||
out_path = Path(out_path)
|
||||
|
||||
try:
|
||||
subprocess.run(
|
||||
[binaries.CCExtractor, "-trim", "-nobom", "-noru", "-ru1", "-o", out_path, self.path],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
out_path.unlink(missing_ok=True)
|
||||
if not e.returncode == 10: # No captions found
|
||||
raise
|
||||
def _run_ccextractor() -> bool:
|
||||
try:
|
||||
subprocess.run(
|
||||
[binaries.CCExtractor, "-trim", "-nobom", "-noru", "-ru1", "-o", out_path, self.path],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
out_path.unlink(missing_ok=True)
|
||||
if e.returncode != 10: # 10 = No captions found
|
||||
raise
|
||||
return out_path.exists()
|
||||
|
||||
# Try on the original file first (preserves container-level CC data like c608 boxes),
|
||||
# then fall back to repacked file (ccextractor can fail on some container formats).
|
||||
if not _run_ccextractor():
|
||||
self.repackage()
|
||||
_run_ccextractor()
|
||||
|
||||
if out_path.exists():
|
||||
cc_track = Subtitle(
|
||||
|
||||
@@ -360,9 +360,37 @@ class MultipleChoice(click.Choice):
|
||||
return super(self).shell_complete(ctx, param, incomplete)
|
||||
|
||||
|
||||
class OffsetType(click.ParamType):
|
||||
"""
|
||||
Parses human-friendly time offset strings into milliseconds.
|
||||
|
||||
Accepts: '10s', '500ms', '-5.5s', '200' (bare number = ms).
|
||||
"""
|
||||
|
||||
name = "offset"
|
||||
|
||||
_PATTERN = re.compile(r"^(-?\d+(?:\.\d+)?)\s*(s|ms)?$")
|
||||
|
||||
def convert(
|
||||
self, value: Any, param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None
|
||||
) -> int:
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
value = str(value).strip()
|
||||
m = self._PATTERN.match(value)
|
||||
if not m:
|
||||
self.fail(f"'{value}' is not a valid offset. Use e.g. '10s', '500ms', '-5.5s'.", param, ctx)
|
||||
number = float(m.group(1))
|
||||
unit = m.group(2) or "ms"
|
||||
if unit == "s":
|
||||
return int(number * 1000)
|
||||
return int(number)
|
||||
|
||||
|
||||
SEASON_RANGE = SeasonRange()
|
||||
LANGUAGE_RANGE = LanguageRange()
|
||||
QUALITY_LIST = QualityList()
|
||||
AUDIO_CODEC_LIST = AudioCodecList(Audio.Codec)
|
||||
OFFSET = OffsetType()
|
||||
|
||||
# VIDEO_CODEC_CHOICE will be created dynamically when imported
|
||||
|
||||
Reference in New Issue
Block a user