diff --git a/tests/core/test_anilist_provider.py b/tests/core/test_anilist_provider.py index cc24e682..9d3ef743 100644 --- a/tests/core/test_anilist_provider.py +++ b/tests/core/test_anilist_provider.py @@ -54,9 +54,6 @@ def default_title_language(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(config, "anilist_title_language", "english") -# ---------- reference parsing ---------- - - @pytest.mark.parametrize( ("value", "expected"), [ @@ -82,9 +79,6 @@ def test_parse_anilist_ref(value: Any, expected: Optional[tuple[str, int]]) -> N assert parse_anilist_ref(value) == expected -# ---------- search ---------- - - def test_search_returns_the_best_match(monkeypatch: pytest.MonkeyPatch) -> None: provider, sent = _provider(monkeypatch, {"data": {"Page": {"media": [ONE_PIECE, ONE_PIECE_MOVIE]}}}) @@ -151,9 +145,6 @@ def test_graphql_errors_are_a_miss_not_a_raise(monkeypatch: pytest.MonkeyPatch) assert provider.get_by_id(21, "tv") is None -# ---------- id lookup ---------- - - def test_get_by_id_queries_the_anilist_id(monkeypatch: pytest.MonkeyPatch) -> None: provider, sent = _provider(monkeypatch, {"data": {"Media": ONE_PIECE}}) @@ -189,9 +180,6 @@ def test_get_external_ids_needs_no_lookup_for_a_bare_id(monkeypatch: pytest.Monk assert sent[0]["variables"] == {"idMal": 21} -# ---------- title variant ---------- - - @pytest.mark.parametrize( ("configured", "expected"), [("english", "One Piece"), ("romaji", "ONE PIECE"), ("native", "ONE PIECE"), ("klingon", "One Piece")], @@ -222,9 +210,6 @@ def test_a_title_with_no_variant_at_all_is_a_miss(monkeypatch: pytest.MonkeyPatc assert provider.get_by_id(21, "tv") is None -# ---------- derived fields ---------- - - @pytest.mark.parametrize( ("country", "language"), [("JP", "ja"), ("KR", "ko"), ("CN", "zh"), ("TW", "zh"), ("US", None), (None, None)], @@ -258,9 +243,6 @@ def test_the_provider_needs_no_api_key() -> None: assert AniListProvider.ID_KIND == "anilist" -# ---------- cache round-trip ---------- - - def test_a_cached_node_rebuilds_with_the_read_time_title_language(monkeypatch: pytest.MonkeyPatch) -> None: """The raw node is cached; which title variant it yields follows the config at read time.""" from unshackle.core.providers import cached_to_result diff --git a/tests/core/test_click_types.py b/tests/core/test_click_types.py index fe2c95b7..eee5de87 100644 --- a/tests/core/test_click_types.py +++ b/tests/core/test_click_types.py @@ -66,9 +66,6 @@ def test_slow_delay_range_accepts_bool(value, expected): assert SLOW_DELAY_RANGE.convert(value, None, None) == expected -# --- LanguageRange ---------------------------------------------------------- - - @pytest.mark.parametrize( "value,expected", [ @@ -83,9 +80,6 @@ def test_language_range_passes_exclusion_tokens_through(value, expected): assert LANGUAGE_RANGE.convert(value) == expected -# --- SeasonRange ------------------------------------------------------------ - - def parse(*tokens: str) -> list[str]: return SeasonRange().parse_tokens(*tokens) @@ -156,9 +150,6 @@ def test_bad_part_tokens_fail(token): parse(token) -# --- SeasonRange date tokens ------------------------------------------------ - - def test_single_date_token_is_its_own_key(): assert parse("2026-08-11") == ["2026-08-11"] diff --git a/tests/core/test_debug_logging.py b/tests/core/test_debug_logging.py index 7041f160..cdbd6064 100644 --- a/tests/core/test_debug_logging.py +++ b/tests/core/test_debug_logging.py @@ -51,9 +51,6 @@ def debug_log_with_keys(tmp_path: Path): close_debug_logger() -# ---------- JSONL output shape ---------- - - def test_every_line_is_valid_json_with_required_fields(debug_log: Path): log_event("sample_event", message="hi", context={"a": 1}) close_debug_logger() # flush session_end @@ -72,9 +69,6 @@ def test_session_id_is_stable_across_entries(debug_log: Path): assert len(ids) == 1 -# ---------- log_event ---------- - - def test_log_event_emits_with_level_message_and_context(debug_log: Path): log_event("feature_done", level="INFO", message="did it", context={"count": 3}) e = find(read_entries(debug_log), "feature_done") @@ -89,9 +83,6 @@ def test_log_event_noop_when_disabled(): log_event("should_not_explode", message="no logger") # must not raise -# ---------- timed_operation ---------- - - def test_timed_operation_logs_success_and_duration(debug_log: Path): with timed_operation("work", level="INFO", context={"k": "v"}): pass @@ -120,9 +111,6 @@ def test_timed_operation_noop_when_disabled(): assert value == 2 -# ---------- secret redaction / key gating ---------- - - def test_sanitize_redacts_sensitive_key_names(debug_log: Path): dl = get_debug_logger() cleaned = dl.sanitize_data( @@ -195,9 +183,6 @@ def test_logged_content_url_is_collapsed_on_disk(debug_log: Path): assert "redacted.mp4" in raw -# ---------- redact.py helpers ---------- - - @pytest.mark.parametrize( ("text", "expected"), [ @@ -221,9 +206,6 @@ def test_safe_display_url_strips_userinfo_and_query(): assert safe_display_url("https://u:p@host:8080/a/b?token=x&y=2") == "https://host:8080/a/b" -# ---------- redact_url ---------- - - @pytest.mark.parametrize( ("url", "expected"), [ @@ -246,9 +228,6 @@ def test_redact_url_handles_url_embedded_in_text(): assert redact_url("got https://cdn.x/a.mp4 ok") == "got redacted.mp4 ok" -# ---------- redact_path ---------- - - def test_redact_path_strips_install_root(): # redact.py resolves the project/install root as the anchor. from unshackle.core.utils import redact as redact_mod @@ -279,9 +258,6 @@ def test_redact_path_disabled_by_config(monkeypatch): assert redact_path(full) == "/temp/Audio.mp4" -# ---------- redact_all + output_dir integration ---------- - - def test_redact_all_composes_secrets_urls_and_paths(): root = str(Path(__import__("unshackle.core.utils.redact", fromlist=["x"]).__file__).resolve().parents[3]) out = redact_all(f"dl https://u:p@cdn.x/seg.mp4?token=t into {root}/temp") @@ -298,9 +274,6 @@ def test_logged_output_dir_is_path_redacted(debug_log: Path): assert e["context"]["filename"] == "Audio_3eeab4ed.mp4" # bare filename untouched -# ---------- convenience-helper message override (footgun regression) ---------- - - def test_log_drm_operation_accepts_message_and_level_override(debug_log: Path): dl = get_debug_logger() # Previously raised: TypeError: log() got multiple values for keyword argument 'message' @@ -332,9 +305,6 @@ def test_log_service_call_accepts_message_override(debug_log: Path): assert e["request"]["status"] == 200 -# ---------- tool_run ---------- - - def test_log_tool_run_success_is_debug(debug_log: Path): log_tool_run("ffmpeg concat", "ffmpeg", 0, duration_ms=12.3, segments=5) e = find(read_entries(debug_log), "tool_run") diff --git a/tests/core/test_episode_absolute.py b/tests/core/test_episode_absolute.py index 52b21f35..1adea2b7 100644 --- a/tests/core/test_episode_absolute.py +++ b/tests/core/test_episode_absolute.py @@ -46,9 +46,6 @@ def reset_template(): config.output_template, config.folder_templates, config.folder_template, config.tag = saved -# --- validation ------------------------------------------------------------- - - def test_absolute_defaults_to_none(): assert make_episode().absolute is None @@ -86,9 +83,6 @@ def test_old_cache_entry_reads_as_absolute_less(): assert restored.absolute is None -# --- naming ----------------------------------------------------------------- - - def test_absolute_token_is_three_wide(reset_template): assert make_episode(absolute=7).build_template_context(StubMediaInfo())["absolute"] == "007" @@ -122,9 +116,6 @@ def test_str_and_sorting_ignore_the_absolute_number(): assert [e.number for e in series] == [1, 2] -# --- the enrich fill -------------------------------------------------------- - - class _FakeProvider: def __init__(self, mapping: dict, source_order: str = "official") -> None: self.mapping = mapping diff --git a/tests/core/test_episode_air_date.py b/tests/core/test_episode_air_date.py index 68abed45..fa6135dd 100644 --- a/tests/core/test_episode_air_date.py +++ b/tests/core/test_episode_air_date.py @@ -122,9 +122,6 @@ def test_dated_folder_keeps_distinct_year(reset_template): assert "2013" in folder -# --- selection by date ------------------------------------------------------ - - def test_a_dated_episode_matches_its_iso_date_key(): assert make_episode(air_date=date(2024, 6, 30)).matches_wanted({"2024-06-30"}) @@ -147,9 +144,6 @@ def test_an_undated_episode_is_unaffected_by_date_keys(): assert episode.matches_wanted({"2024x181"}) -# --- the air-date guard ----------------------------------------------------- - - def test_a_pre_1970_air_date_is_rejected(): with pytest.raises(ValueError): make_episode(air_date=date(1969, 12, 31)) diff --git a/tests/core/test_episode_part.py b/tests/core/test_episode_part.py index d166c31f..6bfc8e86 100644 --- a/tests/core/test_episode_part.py +++ b/tests/core/test_episode_part.py @@ -44,9 +44,6 @@ def reset_template(): config.output_template, config.folder_templates, config.folder_template, config.tag = saved -# --- validation ------------------------------------------------------------- - - def test_part_defaults_to_none(): assert make_episode().part is None @@ -79,9 +76,6 @@ def test_old_cache_entry_reads_as_part_less(): assert restored.part is None -# --- display ---------------------------------------------------------------- - - def test_str_is_unchanged_without_a_part(): without = make_episode() assert str(without) == "The Show S01E01 The Reckoning" @@ -125,9 +119,6 @@ def test_series_sorts_parts_within_their_episode(): assert [(e.number, e.part) for e in series] == [(1, None), (1, 1), (1, 2), (2, None)] -# --- naming ----------------------------------------------------------------- - - def test_filename_is_unchanged_without_a_part(reset_template): assert make_episode().get_filename(StubMediaInfo()) == "The.Show.S01E01.DummyService-GROUP" @@ -168,8 +159,6 @@ def test_parts_share_one_derived_season_folder(reset_template): assert folders == {make_episode().get_filename(StubMediaInfo(), folder=True)} -# --- matching --------------------------------------------------------------- - # the worked key scheme from the design: (token, part-less E1 selected, E1 part 2 selected) MATCH_TABLE = [ (("S01",), True, True), diff --git a/tests/core/test_lang_exclusions.py b/tests/core/test_lang_exclusions.py index 8dd8be4a..69f6760f 100644 --- a/tests/core/test_lang_exclusions.py +++ b/tests/core/test_lang_exclusions.py @@ -8,8 +8,6 @@ from langcodes import Language from unshackle.core.utilities import excluded_language_tags, partition_exclusions from unshackle.core.utils.click_types import LANGUAGE_RANGE -# --- partition_exclusions --------------------------------------------------- - @pytest.mark.parametrize( "tokens,expected", @@ -43,9 +41,6 @@ def test_partition_reads_the_tokens_language_range_produces(): assert partition_exclusions(LANGUAGE_RANGE.convert("all,-es;-fr")) == (["all"], ["es", "fr"]) -# --- excluded_language_tags --------------------------------------------------- - - def test_close_match_excludes_the_regional_variants(): assert excluded_language_tags(["es"], ["es-419", "es-ES", "en-US"]) == {"es-419", "es-ES"} diff --git a/tests/core/test_metadata_id_validation.py b/tests/core/test_metadata_id_validation.py index db13890e..d4803d03 100644 --- a/tests/core/test_metadata_id_validation.py +++ b/tests/core/test_metadata_id_validation.py @@ -20,9 +20,6 @@ def with_keys(monkeypatch: pytest.MonkeyPatch, **keys: str) -> None: monkeypatch.setattr(config, name, keys.get(name, "")) -# ---------- one ID at a time ---------- - - @pytest.mark.parametrize( ("tmdb", "imdb", "tvdb"), [(27205, "tt1375666", None), (27205, None, 73871), (None, "tt1375666", 73871), (27205, "tt1375666", 73871)], @@ -42,9 +39,6 @@ def test_imdb_alone_passes_on_the_keyless_provider(monkeypatch: pytest.MonkeyPat assert validate_metadata_ids(None, "tt1375666", None) is None -# ---------- the ID must have a provider that can resolve it ---------- - - def test_tmdb_without_its_key_names_the_key(monkeypatch: pytest.MonkeyPatch) -> None: with_keys(monkeypatch) with pytest.raises(click.UsageError) as exc: @@ -88,9 +82,6 @@ def test_one_kind_keeping_the_provider_is_enough(monkeypatch: pytest.MonkeyPatch assert validate_metadata_ids(None, None, 73871) is None -# ---------- the REST path rejects the same combinations ---------- - - @pytest.mark.parametrize( "payload", [ @@ -114,9 +105,6 @@ def test_api_still_reports_a_malformed_id_first() -> None: assert err and "positive integer" in err -# ---------- --anilist stays outside the mutual exclusion ---------- - - @pytest.mark.parametrize("value", [21, "21", "mal:21"]) def test_anilist_alone_passes_without_any_key(monkeypatch: pytest.MonkeyPatch, value: object) -> None: with_keys(monkeypatch) diff --git a/tests/core/test_resolve_by_ids.py b/tests/core/test_resolve_by_ids.py index a9821c52..bffa03f5 100644 --- a/tests/core/test_resolve_by_ids.py +++ b/tests/core/test_resolve_by_ids.py @@ -131,9 +131,6 @@ def test_flat_list_config_applies_to_both_kinds(monkeypatch: pytest.MonkeyPatch) assert [cls.NAME for cls in providers.provider_order()] == ["tvdb", "tmdb"] -# ---------- the anilist namespace ---------- - - def test_an_anilist_id_reaches_the_anilist_provider(monkeypatch: pytest.MonkeyPatch) -> None: anilist = FakeProvider( "anilist", @@ -192,9 +189,6 @@ def test_anilist_combines_with_a_western_id(monkeypatch: pytest.MonkeyPatch) -> assert result.external_ids.anilist_id == 21 -# ---------- the anime hint ---------- - - def test_anime_puts_anilist_first_with_the_rest_behind_it() -> None: ordered = [cls.NAME for cls in providers.provider_order("tv", anime=True)] assert ordered[0] == "anilist" diff --git a/tests/core/test_tvdb.py b/tests/core/test_tvdb.py index 571ffaa3..2e898dc9 100644 --- a/tests/core/test_tvdb.py +++ b/tests/core/test_tvdb.py @@ -194,9 +194,6 @@ def test_get_episodes_does_not_cache_an_empty_listing(monkeypatch: pytest.Monkey assert p._episodes == {} -# --- apply_tvdb_order over multi-part episodes ------------------------------- - - class _Svc: pass diff --git a/tests/remote/unit/test_api_episode_part.py b/tests/remote/unit/test_api_episode_part.py index fdcf21a5..aa057869 100644 --- a/tests/remote/unit/test_api_episode_part.py +++ b/tests/remote/unit/test_api_episode_part.py @@ -82,9 +82,6 @@ def selected(result: Dict[str, Any]) -> List[str]: return keys -# ---------- serialize_title: the JSON contract ---------- - - def test_partless_episode_json_has_no_part_key() -> None: d = serialize_title(ep(part=None)) assert "part" not in d @@ -137,9 +134,6 @@ def test_movie_json_unaffected() -> None: assert "part" not in d -# ---------- list_tracks_handler: the REST wanted filter ---------- - - @pytest.fixture def split_titles() -> List[Episode]: """E1 split into three parts, E2 unsplit.""" @@ -172,9 +166,6 @@ def test_rest_part_of_unsplit_episode_selects_nothing(monkeypatch, split_titles) assert result["status"] == 404 -# ---------- discrete season/episode/part request fields ---------- - - def test_discrete_fields_without_part_select_every_part(monkeypatch, split_titles) -> None: result = run_list_tracks(monkeypatch, split_titles, season=1, episode=1) assert selected(result) == ["1x1.1", "1x1.2", "1x1.3"] @@ -207,18 +198,12 @@ def test_part_is_a_transport_key_not_a_service_option() -> None: assert "part" in handlers.SESSION_TRANSPORT_KEYS -# ---------- ordering ---------- - - def test_rest_multi_episode_order_is_stable_across_parts(monkeypatch) -> None: titles = [ep(1, 2, None), ep(1, 1, 3), ep(1, 1, 1), ep(1, 1, 2)] result = run_list_tracks(monkeypatch, titles, wanted="S01") assert selected(result) == ["1x1.1", "1x1.2", "1x1.3", "1x2"] -# ---------- download_manager: already-internal wanted sniff ---------- - - SNIFF = r"^!?\d+x\d+(\.\d+)?$" @@ -283,9 +268,6 @@ def test_perform_download_still_converts_cli_style_tokens(monkeypatch) -> None: assert params["wanted"] == ["1x1"] -# ---------- CLI / REST agreement ---------- - - def test_dl_and_handlers_share_the_one_match_helper() -> None: """Both filters must route through Episode.matches_wanted, or they drift.""" from pathlib import Path diff --git a/tests/remote/unit/test_api_title_fields.py b/tests/remote/unit/test_api_title_fields.py index 874eadac..0ec4495d 100644 --- a/tests/remote/unit/test_api_title_fields.py +++ b/tests/remote/unit/test_api_title_fields.py @@ -49,9 +49,6 @@ def _movie(**overrides: Any) -> Movie: return Movie(**base) -# ---------- serialize_title: the new conditional keys ---------- - - NEW_KEYS = ("air_date", "absolute", "daily", "anime") @@ -94,9 +91,6 @@ def test_a_movie_carries_the_anime_flag_but_never_daily() -> None: assert "daily" not in d -# ---------- the service-class ANIME/DAILY stamp ---------- - - class _AnimeDailySvc: ANIME = True DAILY = True @@ -149,9 +143,6 @@ def test_list_titles_stamps_the_service_flags(monkeypatch: pytest.MonkeyPatch) - assert (title["anime"], title["daily"]) == (True, True) -# ---------- round trips through build_title ---------- - - def round_trip(episode: Episode, stamp: Optional[Any] = None) -> Episode: info = serialize_title(episode) if stamp is not None: @@ -196,9 +187,6 @@ def test_a_movie_round_trip_keeps_the_anime_flag() -> None: assert rebuilt.anime is True -# ---------- round trips through title_to_meta (the export/import path) ---------- - - def test_title_to_meta_omits_the_new_keys_for_a_plain_episode() -> None: meta = dl.title_to_meta(ep()) assert not [k for k in ("absolute", "daily", "anime") if k in meta] diff --git a/tests/remote/unit/test_download_gates_redaction.py b/tests/remote/unit/test_download_gates_redaction.py index 91668ab3..a76ce17f 100644 --- a/tests/remote/unit/test_download_gates_redaction.py +++ b/tests/remote/unit/test_download_gates_redaction.py @@ -21,9 +21,6 @@ from unshackle.core.api.errors import APIError, APIErrorCode pytestmark = pytest.mark.unit -# ---------- redaction ---------- - - def test_redact_parameters_masks_secrets_and_proxy_userinfo(): params = { "service": "ATV", @@ -85,9 +82,6 @@ def test_to_dict_full_details_redacts_error_fields_and_parameters(): assert d["parameters"]["credential"] == "***" -# ---------- gates ---------- - - class _PastGate(Exception): """Raised by the stubbed Services.load to prove a request got past the gate into the try block.""" @@ -140,9 +134,6 @@ async def test_credential_allowed_when_enabled(stub_handler): assert isinstance(resp, web.Response) -# ---------- range validation ---------- - - def test_range_validation_accepts_hdr10p_and_alias(): # canonical "HDR10P" and back-compat "HDR10+" both pass; mixed casing too assert handlers.validate_download_parameters({"range": ["HDR10P", "DV", "SDR"]}) is None diff --git a/tests/remote/unit/test_errors.py b/tests/remote/unit/test_errors.py index e0de7dfc..e8f18b0e 100644 --- a/tests/remote/unit/test_errors.py +++ b/tests/remote/unit/test_errors.py @@ -142,7 +142,6 @@ def test_handle_api_exception_categorizes_generic() -> None: assert body["error_code"] == "NETWORK_ERROR" -# --- Error contract harness (PRD 05 gate) --------------------------------- # Locks the FULL observable response — (http status, error_code, retryable) — that # a client sees for a generic (non-APIError) exception flowing through the # handle_api_exception funnel that every route's `except Exception` path calls. diff --git a/tests/remote/unit/test_handlers_serialize.py b/tests/remote/unit/test_handlers_serialize.py index 0fe340d2..e04959ca 100644 --- a/tests/remote/unit/test_handlers_serialize.py +++ b/tests/remote/unit/test_handlers_serialize.py @@ -77,9 +77,6 @@ def subtitle(**overrides) -> Subtitle: return Subtitle(**base) -# ---------- sanitize_log ---------- - - @pytest.mark.parametrize( "raw, expected", [ @@ -93,9 +90,6 @@ def test_sanitize_log(raw, expected: str) -> None: assert sanitize_log(raw) == expected -# ---------- serialize_title ---------- - - def test_serialize_title_movie() -> None: movie = Movie(id_="movie-0001", service=_FakeSvc, name="Title X", year=2024, language=Language.get("en")) d = serialize_title(movie) @@ -140,9 +134,6 @@ def test_serialize_title_episode_unnamed_falls_back_to_number() -> None: assert d["name"] == "Episode 05" -# ---------- serialize_video/audio/subtitle ---------- - - def test_serialize_video_track_basic() -> None: d = serialize_video_track(video()) assert d["id"] == "video-001" @@ -179,9 +170,6 @@ def test_serialize_subtitle_track_basic() -> None: assert d["cc"] is False -# ---------- serialize_drm ---------- - - def test_serialize_drm_none_returns_none() -> None: assert serialize_drm(None) is None assert serialize_drm([]) is None @@ -224,16 +212,10 @@ def test_serialize_drm_playready_pssh_without_dumps_is_omitted() -> None: assert info["kids"] == ["00112233445566778899aabbccddeeff"] -# ---------- validate_service ---------- - - def test_validate_service_unknown_returns_none() -> None: assert validate_service("NOPE_THIS_IS_NOT_REAL_") is None -# ---------- validate_download_parameters ---------- - - def test_validate_download_params_accepts_defaults() -> None: assert validate_download_parameters({}) is None diff --git a/tests/remote/unit/test_progress_sink.py b/tests/remote/unit/test_progress_sink.py index 2d1175c9..e0886365 100644 --- a/tests/remote/unit/test_progress_sink.py +++ b/tests/remote/unit/test_progress_sink.py @@ -19,7 +19,6 @@ from unshackle.core.api.progress import ( pytestmark = pytest.mark.unit -# --- lightweight track stand-ins (label/weight key off class name + attributes) --- class _Range: def __init__(self, value): self.value = value diff --git a/tests/tracks/test_hybrid_selection.py b/tests/tracks/test_hybrid_selection.py index 5f128d09..a3b49806 100644 --- a/tests/tracks/test_hybrid_selection.py +++ b/tests/tracks/test_hybrid_selection.py @@ -58,11 +58,6 @@ def ids(tracks: list[Video]) -> set[str]: return {t.id for t in tracks} -# --------------------------------------------------------------------------- -# select_hybrid -# --------------------------------------------------------------------------- - - def test_select_hybrid_picks_base_per_resolution_and_lowest_dv(ladder: list[Video]) -> None: chosen = list(filter(Tracks().select_hybrid(ladder, [2160, 1080]), ladder)) # HDR10+ base at each requested resolution, plus the single lowest DV ingredient. @@ -107,11 +102,6 @@ def test_select_hybrid_no_dv_selects_only_base() -> None: assert ids(chosen) == {"hdr10p-2160"} -# --------------------------------------------------------------------------- -# merge_video_selections (the dedup fix) -# --------------------------------------------------------------------------- - - def test_merge_dedups_shared_ingredient_and_deliverable() -> None: H = Video.Codec.HEVC dv = make_video("dv-1080", range_=Video.Range.DV, height=1080, bitrate=9_000_000, codec=H) @@ -141,11 +131,6 @@ def test_merge_dedup_uses_track_identity_by_id() -> None: assert len(Tracks.merge_video_selections([a], [b])) == 1 -# --------------------------------------------------------------------------- -# partition_hybrid_videos -# --------------------------------------------------------------------------- - - def test_partition_hybrid_only_keeps_ingredients_out_of_pool(ladder: list[Video]) -> None: """`-r HYBRID`: HDR10+/DV are ingredients only; pool holds just SDR.""" candidates, pool = Tracks.partition_hybrid_videos(ladder, []) @@ -178,11 +163,6 @@ def test_partition_hdr10_requested_does_not_admit_hdr10p() -> None: assert ids(pool) == {"hdr10-2160"} -# --------------------------------------------------------------------------- -# flag_hybrid_ingredients -# --------------------------------------------------------------------------- - - def flagged(tracks: list[Video]) -> set[str]: return {t.id for t in tracks if t.hybrid_base_only} @@ -228,11 +208,6 @@ def test_flag_single_dv_rendition_as_deliverable_stays_unflagged() -> None: assert flagged(tracks) == {"hdr10p-1080"} -# --------------------------------------------------------------------------- -# documented end-state for the reported command -# --------------------------------------------------------------------------- - - def test_hybrid_plus_dv_deliverable_end_state(ladder: list[Video]) -> None: """`-r SDR,HDR10P,DV,HYBRID -q 2160,1080`: hybrid base + lowest DV ingredient, de-duplicated against the best-DV-per-resolution deliverables and SDR.""" diff --git a/tests/tracks/test_subtitle_convert.py b/tests/tracks/test_subtitle_convert.py index aa220dde..6fad00e7 100644 --- a/tests/tracks/test_subtitle_convert.py +++ b/tests/tracks/test_subtitle_convert.py @@ -67,9 +67,6 @@ def cue_count(path) -> int: return len(re.findall(r"-->", path.read_text("utf8"))) -# --- capability matrix / resolver ------------------------------------------------------- - - def test_resolve_webvtt_to_srt_order(): chain = [b.name for b in sc.resolve_backends(Codec.WebVTT, Codec.SubRip)] assert chain == ["subby", "pysubs2", "pycaption"] @@ -131,9 +128,6 @@ def test_no_backend_for_unsupported_target_raises(tmp_path): sub.convert(Codec.fVTT, forced=True) # no backend writes segmented fVTT -# --- styled-ASS protection -------------------------------------------------------------- - - def test_ass_to_srt_kept_as_is_when_not_forced(tmp_path, monkeypatch): monkeypatch.setattr("unshackle.core.config.config.subtitle", {}, raising=False) sub = make_sub(tmp_path, "x.ass", ASS_SAMPLE, Codec.SubStationAlphav4) @@ -153,9 +147,6 @@ def test_ass_to_srt_converts_when_forced(tmp_path, monkeypatch): assert "{\\" not in out.read_text("utf8") # override tags stripped -# --- conversion paths ------------------------------------------------------------------- - - def test_webvtt_to_srt_conversion(tmp_path, monkeypatch): monkeypatch.setattr("unshackle.core.config.config.subtitle", {}, raising=False) sub = make_sub(tmp_path, "x.vtt", VTT_SAMPLE, Codec.WebVTT) @@ -170,8 +161,6 @@ def test_same_codec_is_noop(tmp_path): assert sub.codec == Codec.SubRip -# --- ASS/SSA font detection ------------------------------------------------------------ - FONT_ASS = """[Script Info] ScriptType: v4.00+ @@ -205,8 +194,6 @@ def test_extract_fonts_handles_non_default_column_order(): assert Subtitle.extract_fonts(ass) == {"Verdana"} -# --- non-Latin scripts (RTL / CJK) preserved through conversion ------------------------ - CJK_RTL_VTT = """WEBVTT 1 @@ -236,8 +223,6 @@ def test_non_latin_scripts_survive_vtt_to_srt(tmp_path, monkeypatch, pattern): assert re.search(pattern, text) # script survived the round-trip, no mojibake -# --- SDH stripping ---------------------------------------------------------------------- - SDH_SRT = """1 00:00:01,000 --> 00:00:02,000 [door creaks] @@ -293,9 +278,8 @@ def test_sdh_stripping_ass_without_subtitleedit_content_matches_codec(tmp_path, assert "door creaks" not in text # bracketed effect removed -# --- segmented (box-encapsulated) formats: fVTT (wvtt) / fTTML (stpp) -------------------- -# These ship from DASH/HLS as fragmented MP4 (e.g. HBO Max). The downloader concatenates -# init + media segments into one file; parse() reads the MP4 boxes directly. +# Segmented fVTT (wvtt) / fTTML (stpp) ship from DASH/HLS as fragmented MP4 (e.g. HBO Max). The +# downloader concatenates init + media segments into one file; parse() reads the MP4 boxes directly. FIXTURES = pathlib.Path(__file__).parent / "fixtures" diff --git a/tests/tracks/test_track_selection_fixes.py b/tests/tracks/test_track_selection_fixes.py index f99ad9de..aefbbbc3 100644 --- a/tests/tracks/test_track_selection_fixes.py +++ b/tests/tracks/test_track_selection_fixes.py @@ -55,9 +55,7 @@ def make_audio( ) -# --------------------------------------------------------------------------- # by_resolutions: width=None must not crash the canvas fallback -# --------------------------------------------------------------------------- def test_by_resolutions_tolerates_width_none_track() -> None: @@ -73,9 +71,7 @@ def test_by_resolutions_tolerates_width_none_track() -> None: assert [t.id for t in tracks.videos] == ["canvas-720"] -# --------------------------------------------------------------------------- # sort_videos: resolution before bitrate -# --------------------------------------------------------------------------- def test_sort_videos_resolution_outranks_bitrate() -> None: @@ -102,9 +98,7 @@ def test_sort_videos_unknown_bitrate_still_wins_on_resolution() -> None: assert [t.id for t in tracks.videos] == ["unknown-1080", "known-720"] -# --------------------------------------------------------------------------- # sort_audio: first track after sort is the "best" the dl layer picks -# --------------------------------------------------------------------------- def test_sort_audio_atmos_ranks_first() -> None: diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index 0960d693..3a672372 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -315,17 +315,14 @@ class dl: Returns: Path to temp font file """ - # Find the matched name for logging matched_name = next((name for name, path in system_fonts.items() if path == matched_font), None) if matched_name and matched_name.lower() != font_name.lower(): self.log.info(f"Using '{matched_name}' as fallback for '{font_name}'") - # Create unique temp file path safe_name = font_name.replace(" ", "_").replace("/", "_") temp_path = config.directories.temp / f"font_{safe_name}{matched_font.suffix}" - # Copy if not already exists if not temp_path.exists(): shutil.copy2(matched_font, temp_path) temp_font_files.append(temp_path) @@ -352,13 +349,11 @@ class dl: missing_fonts = [] for font_name in set(font_names): - # Try custom fonts first if custom_font := self.find_custom_font(font_name): title.tracks.add(Attachment(path=custom_font, name=f"{font_name} ({custom_font.stem})")) font_count += 1 continue - # Try system fonts with fallback if system_font := find_font_with_fallbacks(font_name, system_fonts): temp_path = self.prepare_temp_font(font_name, system_font, system_fonts, temp_font_files) title.tracks.add(Attachment(path=temp_path, name=f"{font_name} ({system_font.stem})")) @@ -410,7 +405,11 @@ class dl: sidecar_format: str, original_paths: Optional[dict[str, Path]] = None, ) -> list[Path]: - """Output subtitles as sidecar files, converting if needed.""" + """ + Output subtitles as sidecar files, converting if needed. + + Conversion runs on a temp copy, so each subtitle's own file is left untouched and can still be muxed. + """ created_paths: list[Path] = [] config.directories.temp.mkdir(parents=True, exist_ok=True) @@ -422,7 +421,6 @@ class dl: if not source_path or not source_path.exists(): continue - # Determine target codec if sidecar_format == "original": target_codec = None if source_path.suffix: @@ -437,11 +435,9 @@ class dl: subtitle, base_filename, output_dir, target_codec, source_path=source_path ) - # Copy or convert if not target_codec or subtitle.codec == target_codec: shutil.copy2(source_path, sidecar_path) else: - # Create temp copy for conversion to preserve original temp_path = config.directories.temp / f"sidecar_{subtitle.id}{source_path.suffix}" shutil.copy2(source_path, temp_path) @@ -1505,7 +1501,6 @@ class dl: else: export_path = None - # Parse bitrate range options vbitrate_min, vbitrate_max = None, None if vbitrate_range: if vbitrate and vbitrate_range: @@ -1559,7 +1554,6 @@ class dl: self.log.error("--select-titles and -w/--wanted cannot be used together") sys.exit(1) - # Check if dovi_tool is available when hybrid mode is requested if any(r == Video.Range.HYBRID for r in range_): from unshackle.core.binaries import DoviTool @@ -1796,12 +1790,9 @@ class dl: selection_titles.append(header_text) current_season_header_idx = len(selection_titles) - 1 dependencies[current_season_header_idx] = [] - # Note: Headers are not mapped to actual title indices - # Format display name display_name = ((t.name[:30].rstrip() + "...") if len(t.name) > 30 else t.name) if t.name else None - # Apply indentation only for multiple seasons prefix = " " if multiple_seasons else "" part_label = f".{t.part}" if t.part is not None else "" option_text = f"{prefix}{t.number}{part_label}" + (f". {display_name}" if t.name else "") @@ -1809,16 +1800,13 @@ class dl: selection_titles.append(option_text) current_ui_idx = len(selection_titles) - 1 - # Map UI index to actual title index original_indices[current_ui_idx] = i - # Link episode to season header for group selection if current_season_header_idx != -1: dependencies[current_season_header_idx].append(current_ui_idx) selection_start = time.time() - # Execute selector with dependencies (headers select all children) selected_ui_idx = select_multiple( selection_titles, minimal_count=1, @@ -1835,7 +1823,6 @@ class dl: selection_end = time.time() start_time += selection_end - selection_start - # Map UI indices back to title indices (excluding headers) selected_idx = [] for idx in selected_ui_idx: if idx in original_indices: @@ -1845,17 +1832,14 @@ class dl: selected_idx = sorted(set(selected_idx)) keep = set(selected_idx) - # In-place filter: remove unselected items (iterate backwards) for i in range(len(titles) - 1, -1, -1): if i not in keep: del titles[i] - # Show selected count if titles: count = len(titles) console.print(Padding(f"[text]Total selected: {count}[/]", (0, 5))) - # Determine the latest episode if --latest-episode is set latest_episode_id = None if latest_episode and isinstance(titles, Series) and len(titles) > 0: # Series is already sorted by (season, number, year) @@ -2625,7 +2609,6 @@ class dl: ) continue - # Determine which tracks to keep keep_videos = True keep_audio = True keep_subtitles = True @@ -2663,7 +2646,6 @@ class dl: if no_proxy_download and any(service.session.proxies.values()): console.log("Bypassing proxy for downloads as --no-proxy-download was used...") - # Clear unwanted tracks if not keep_videos: title.tracks.videos = [] if not keep_audio: @@ -2692,7 +2674,6 @@ class dl: fsl_excl_r = resolve_excludes(fsl_excl) if isinstance(title, (Movie, Episode)): - # filter video tracks if keep_videos and vcodec: title.tracks.select_video(lambda x: x.codec in vcodec) missing_codecs = [c for c in vcodec if not any(x.codec == c for x in title.tracks.videos)] @@ -2876,10 +2857,8 @@ class dl: self.log.error(f"There's no {res_list} Video Track{plural}...") sys.exit(1) - # choose best track by range and quality pre_hybrid_videos: list[Video] = list(title.tracks.videos) if has_hybrid else [] if keep_videos and has_hybrid: - # Apply hybrid selection for HYBRID tracks hybrid_candidate_tracks, non_hybrid_tracks = Tracks.partition_hybrid_videos( title.tracks.videos, non_hybrid_ranges ) @@ -2899,10 +2878,8 @@ class dl: ) hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks)) - # For non-hybrid ranges, apply Cartesian product selection non_hybrid_selected: list[Video] = [] if non_hybrid_ranges and non_hybrid_tracks: - # Include language dimension when multiple video languages were requested if video_multi_lang: non_hybrid_langs = list(dict.fromkeys(str(v.language) for v in non_hybrid_tracks)) else: @@ -2958,7 +2935,6 @@ class dl: selected_videos.append(match) title.tracks.videos = selected_videos - # validate hybrid mode requirements if keep_videos and any(r == Video.Range.HYBRID for r in range_): base_tracks = [ v for v in title.tracks.videos if v.range in (Video.Range.HDR10, Video.Range.HDR10P) @@ -3022,7 +2998,6 @@ class dl: self.log.error(msg_detail) sys.exit(1) - # filter subtitle tracks fsl = [t for t in forced_s_lang if t != "orig"] if "orig" in forced_s_lang and title.language: fsl.append(str(title.language)) @@ -3105,11 +3080,10 @@ class dl: ) title.tracks.select_subtitles(lambda x: not (x.forced and str(x.language) in drop)) - # filter audio tracks # might have no audio tracks if part of the video, e.g. transport stream hls if keep_audio and len(title.tracks.audio) > 0: if not audio_description: - title.tracks.select_audio(lambda x: not x.descriptive) # exclude descriptive audio + title.tracks.select_audio(lambda x: not x.descriptive) if acodec: title.tracks.select_audio(lambda x: x.codec in acodec) if not title.tracks.audio: @@ -3208,7 +3182,6 @@ class dl: ) sys.exit(1) - # Reconstruct track set to only include kept tracks kept_tracks = [] if keep_videos: kept_tracks.extend(title.tracks.videos) @@ -3411,7 +3384,6 @@ class dl: dl_time = time_elapsed_since(dl_start_time) console.print(Padding(f"Track downloads finished in [progress.elapsed]{dl_time}[/]", (0, 5))) - # Subtitle output mode configuration (for sidecar originals) subtitle_output_mode = config.subtitle.get("output_mode", "mux") sidecar_format = config.subtitle.get("sidecar_format", "srt") skip_subtitle_mux = subtitle_output_mode == "sidecar" and (title.tracks.videos or title.tracks.audio) @@ -3493,7 +3465,6 @@ class dl: if has_decrypted: self.log.info(f"Decrypted tracks with {decrypt_tool}") - # Extract Closed Captions from decrypted video tracks if ( not no_subs and not (hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES) @@ -3562,7 +3533,6 @@ class dl: "decryption likely failed and the muxed output may be unplayable." ) if has_repacked: - # we don't want to fill up the log with "Repacked x track" self.log.info("Repacked one or more tracks with FFMPEG") with console.status("Normalizing video VUI..."): @@ -3579,7 +3549,6 @@ class dl: append_audio_codec_suffix = True if no_mux: - # Skip muxing, handle individual track files for track in title.tracks: if track.path and track.path.exists(): muxed_paths.append(track.path) @@ -3601,7 +3570,6 @@ class dl: # When we split audio (merge_audio=False), multiple outputs may exist per title, so suffix codec. append_audio_codec_suffix = not merge_audio - # Mux all selected video tracks into one file instead of one file per track. merge_video = merge_video if merge_video is not None else config.muxing.get("merge_video", False) multiplex_tasks: list[tuple[TaskID, Tracks, Optional[Audio.Codec]]] = [] @@ -3709,7 +3677,7 @@ class dl: hybrid_track = deepcopy(hdr10_track) hybrid_track.id = f"hybrid_{hdr10_track.id}_{resolution}" hybrid_track.path = hybrid_output_path - hybrid_track.range = Video.Range.DV # It's now a DV track + hybrid_track.range = Video.Range.DV hybrid_track.needs_duration_fix = True title.tracks.add(hybrid_track) task_tracks.videos = [hybrid_track] @@ -3781,7 +3749,6 @@ class dl: if mux_failed: sys.exit(1) - # Output sidecar subtitles before deleting track files if sidecar_subtitles and not no_mux: media_info = MediaInfo.parse(muxed_paths[0]) if muxed_paths else None if media_info: @@ -3817,14 +3784,12 @@ class dl: for track in title.tracks: track.delete() - # Clear temp font attachment paths and delete other attachments for attachment in title.tracks.attachments: if attachment.path and attachment.path in temp_font_files: attachment.path = None else: attachment.delete() - # Clean up temp fonts for temp_path in temp_font_files: temp_path.unlink(missing_ok=True) for temp_path in sidecar_original_paths.values(): @@ -3845,18 +3810,14 @@ class dl: ) else: - # dont mux muxed_paths.append(title.tracks.audio[0].path) if no_mux: - # Handle individual track files without muxing final_dir = self.output_dir or config.directories.downloads if not no_folder and ( isinstance(title, (Episode, Song)) or (isinstance(title, Movie) and config.get_folder_template("movies")) ): - # Create folder based on title - # Use first available track for filename generation sample_track = ( title.tracks.videos[0] if title.tracks.videos @@ -3873,11 +3834,9 @@ class dl: final_dir.mkdir(parents=True, exist_ok=True) for track_path in muxed_paths: - # Generate appropriate filename for each track media_info = MediaInfo.parse(track_path) base_filename = title.get_filename(media_info, show_service=not no_source) - # Add track type suffix to filename track = next((t for t in title.tracks if t.path == track_path), None) if track: if isinstance(track, Video): @@ -3903,7 +3862,6 @@ class dl: self.completed_files.append(final_path) self.log.debug(f"Saved: {final_path.name}") else: - # Handle muxed files used_final_paths: set[Path] = set() for muxed_path in muxed_paths: media_info = MediaInfo.parse(muxed_path) @@ -4929,8 +4887,12 @@ class dl: @staticmethod def get_cookie_jar(service: str, profile: Optional[str]) -> Optional[CookieJar]: - """Get Service Cookies for Profile.""" - # Check if automatic Firefox cookie extraction is configured for this service + """ + Get Service Cookies for Profile. + + If firefox_cookies is configured for the service, cookies come from Firefox first. If that extraction + raises or returns nothing, the profile's cookie file is used instead and the failure is not logged. + """ ff_settings = getattr(config, "firefox_cookies", {}).get(service) if ff_settings: try: @@ -4940,7 +4902,6 @@ class dl: if extracted_jar: return extracted_jar except Exception: - # Fallback to file-based if Firefox extraction fails pass cookie_file = dl.get_cookie_path(service, profile) diff --git a/unshackle/commands/env.py b/unshackle/commands/env.py index f3181dd2..510781a4 100644 --- a/unshackle/commands/env.py +++ b/unshackle/commands/env.py @@ -21,7 +21,6 @@ from unshackle.core.temp import TASK_PREFIX, is_stale def get_dependencies() -> list[dict]: """Binary dependency inventory shared by `env check` and the API env check.""" return [ - # Core Media Tools {"name": "FFmpeg", "binary": binaries.FFMPEG, "required": True, "desc": "Media processing", "cat": "Core"}, {"name": "FFprobe", "binary": binaries.FFProbe, "required": True, "desc": "Media analysis", "cat": "Core"}, {"name": "MKVToolNix", "binary": binaries.MKVToolNix, "required": True, "desc": "MKV muxing", "cat": "Core"}, @@ -46,7 +45,6 @@ def get_dependencies() -> list[dict]: "desc": "DRM decryption", "cat": "DRM", }, - # HDR Processing {"name": "dovi_tool", "binary": binaries.DoviTool, "required": False, "desc": "Dolby Vision", "cat": "HDR"}, { "name": "HDR10Plus_tool", @@ -55,7 +53,6 @@ def get_dependencies() -> list[dict]: "desc": "HDR10+ metadata", "cat": "HDR", }, - # Subtitle Tools { "name": "SubtitleEdit", "binary": binaries.SubtitleEdit, @@ -70,10 +67,8 @@ def get_dependencies() -> list[dict]: "desc": "CC extraction", "cat": "Subtitle", }, - # Media Players {"name": "FFplay", "binary": binaries.FFPlay, "required": False, "desc": "Simple player", "cat": "Player"}, {"name": "MPV", "binary": binaries.MPV, "required": False, "desc": "Advanced player", "cat": "Player"}, - # Network Tools { "name": "HolaProxy", "binary": binaries.HolaProxy, @@ -132,13 +127,11 @@ def check() -> None: """Checks environment for the required dependencies.""" all_deps = get_dependencies() - # Track overall status all_required_installed = True total_installed = 0 total_required = 0 missing_required = [] - # Create a single table table = Table( title="Environment Dependencies", title_style="bold", show_header=True, header_style="bold", expand=False ) @@ -152,11 +145,9 @@ def check() -> None: for dep in all_deps: path = dep["binary"] - # Category column (only show when it changes) category = dep["cat"] if dep["cat"] != last_cat else "" last_cat = dep["cat"] - # Status if path: status = "[green]✓[/green]" total_installed += 1 @@ -169,15 +160,12 @@ def check() -> None: if dep["required"]: total_required += 1 - # Required column (compact) req = "[red]Y[/red]" if dep["required"] else "[bright_black]-[/bright_black]" - # Add row table.add_row(category, dep["name"], status, req, dep["desc"]) console.print(Padding(table, (1, 2))) - # Compact summary summary_parts = [f"[bold]Total:[/bold] {total_installed}/{len(all_deps)}"] if all_required_installed: @@ -321,13 +309,10 @@ def info() -> None: continue attr_value = getattr(config.directories, name) - # Handle both single Path objects and lists of Path objects if isinstance(attr_value, list): - # For lists, show each path on a separate line paths_str = "\n".join(str(p.resolve()) if isinstance(p, Path) else str(p) for p in attr_value) table.add_row(name.title(), paths_str) else: - # For single Path objects, use the original logic path = attr_value.resolve() for var, var_path in path_vars.items(): if path.is_relative_to(var_path): diff --git a/unshackle/commands/search.py b/unshackle/commands/search.py index 3985b4d5..2158f57d 100644 --- a/unshackle/commands/search.py +++ b/unshackle/commands/search.py @@ -89,9 +89,7 @@ def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, pr if proxy: requested_provider = None if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): - # requesting proxy from a specific proxy provider requested_provider, proxy = proxy.split(":", maxsplit=1) - # Match simple region codes (us, ca, uk1) or provider:region format (nordvpn:ca, windscribe:us) if re.match(r"^[a-z]{2}(?:[-][a-z0-9]+)*(?:\d+)?$", proxy, re.IGNORECASE) or re.match( r"^[a-z]+:[a-z]{2}(?:[-][a-z0-9]+)*(?:\d+)?$", proxy, re.IGNORECASE ): @@ -161,7 +159,6 @@ def result(service: Service, profile: Optional[str] = None, **_: Any) -> None: result_text += f"\n[bright_black]id: {result.id}[/]" search_results.add(result_text + "\n") - # update cookies cookie_file = dl.get_cookie_path(service_tag, profile) if cookie_file: dl.save_cookies(cookie_file, service.session.cookies) diff --git a/unshackle/commands/serve.py b/unshackle/commands/serve.py index 3480f9de..049ccf14 100644 --- a/unshackle/commands/serve.py +++ b/unshackle/commands/serve.py @@ -183,7 +183,6 @@ def serve( app["config"] = {"users": api_users} app["debug_api"] = debug_api - # Start session cleanup loop for remote-dl sessions from unshackle.core.api.session_store import get_session_store session_store = get_session_store() @@ -277,7 +276,6 @@ def serve( app["config"] = serve_config app["debug_api"] = debug_api - # Start session cleanup loop for remote-dl sessions from unshackle.core.api.session_store import get_session_store session_store = get_session_store() diff --git a/unshackle/core/api/download_manager.py b/unshackle/core/api/download_manager.py index d71cdb26..affc1716 100644 --- a/unshackle/core/api/download_manager.py +++ b/unshackle/core/api/download_manager.py @@ -91,12 +91,10 @@ class DownloadJob: title_id: str parameters: Dict[str, Any] - # Progress tracking started_time: Optional[datetime] = None completed_time: Optional[datetime] = None progress: float = 0.0 - # Results and error info output_files: List[str] = field(default_factory=list) error_message: Optional[str] = None error_details: Optional[str] = None @@ -104,7 +102,6 @@ class DownloadJob: error_traceback: Optional[str] = None worker_stderr: Optional[str] = None - # Current phase, track counts, and labels of the tracks downloading now. phase: Optional[str] = None title: Optional[str] = None current_title: Optional[str] = None @@ -112,7 +109,6 @@ class DownloadJob: total_tracks: int = 0 active_tracks: List[str] = field(default_factory=list) track_progress: List[Dict[str, Any]] = field(default_factory=list) - # Segment counts and transfer speed of the track downloading now (granular display) segments_done: float = 0.0 segments_total: float = 0.0 speed: Optional[str] = None @@ -121,7 +117,6 @@ class DownloadJob: # dict (id / language / title) so a client can report which weren't available. skipped_subtitles: List[Dict[str, Any]] = field(default_factory=list) - # Cancellation support cancel_event: threading.Event = field(default_factory=threading.Event) # Guards against writing the same job to the persistent history file twice. @@ -255,7 +250,7 @@ def read_job_history(limit: int = 100, service: Optional[str] = None) -> List[Di if service and str(entry.get("service") or "").upper() != service.upper(): continue entries.append(entry) - entries.reverse() # newest first + entries.reverse() return entries[:limit] if limit and limit > 0 else entries @@ -328,7 +323,13 @@ def perform_download( params: Dict[str, Any], progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None, ) -> List[str]: - """Execute the synchronous download logic for a job.""" + """Execute the synchronous download logic for a job. + + `params` holds the API's string form of the `dl` options and is normalized in place before + dl.result() sees it: vcodec and range names become enums, `slow` accepts "MIN-MAX", a two-item + list, or True (which means 60-120), and `wanted` accepts forms such as "S01E01", "S01-S03", + "s1e1", or "1x1" as well as the internal "SxE" form. + """ from contextlib import redirect_stderr, redirect_stdout from io import StringIO @@ -359,7 +360,6 @@ def perform_download( ] config.directories.cache = config.directories.cache / "_jobs" / cred_hash - # Convert string parameters to enums (API receives strings, dl.result() expects enums) vcodec_raw = params.get("vcodec") if vcodec_raw: if isinstance(vcodec_raw, str): @@ -385,7 +385,6 @@ def perform_download( if params.get("export"): params["export"] = bool(params["export"]) - # Normalize slow: accept string "MIN-MAX", list/tuple, or True (default 60-120) slow_raw = params.get("slow") if slow_raw is not None and not isinstance(slow_raw, tuple): if isinstance(slow_raw, bool): @@ -400,22 +399,18 @@ def perform_download( except click.BadParameter as exc: raise Exception(f"Invalid slow parameter: {exc}") - # Convert wanted episode strings to internal "SxE" format - # Accepts: "S01E01", "S01-S03", "s1e1", "1x1", or already-parsed format wanted_raw = params.get("wanted") if wanted_raw: from unshackle.core.utils.click_types import SeasonRange if isinstance(wanted_raw, str): wanted_raw = [wanted_raw] - # Only convert if not already in internal "SxE" format # the !? keeps a pre-parsed part exclusion from being re-fed through parse_tokens needs_conversion = any(not re.match(r"^!?\d+x\d+(\.\d+)?$", w) for w in wanted_raw) if needs_conversion: season_range = SeasonRange() params["wanted"] = season_range.parse_tokens(*wanted_raw) - # Load service configuration service_config_path = Services.get_path(service) / config.filenames.config if service_config_path.exists(): service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) @@ -619,7 +614,6 @@ def perform_download( detail = (stdout_capture.getvalue() + stderr_capture.getvalue())[-200:].strip() raise APIError(APIErrorCode.WORKER_ERROR, "download worker failed: " + (detail or "see logs")) - # Surface any subtitles that were skipped (non-fatal failures) so the client can report them. if progress_callback: skipped_subs = getattr(dl_instance, "skipped_subtitles", None) if skipped_subs: @@ -738,12 +732,10 @@ class DownloadQueueManager: log.info(f"Cancelled queued job {sanitize_log(job_id)}") return True elif job.status == JobStatus.DOWNLOADING: - # Set the cancellation event first - this will be checked by the download thread job.cancel_event.set() job.status = JobStatus.CANCELLED log.info(f"Signaled cancellation for downloading job {sanitize_log(job_id)}") - # Cancel the active download task task = self._active_downloads.get(job_id) if task: task.cancel() @@ -881,7 +873,6 @@ class DownloadQueueManager: if self._priority_jobs: job = self._priority_jobs.popleft() else: - # Wait for a job or shutdown signal job = await asyncio.wait_for(self._job_queue.get(), timeout=1.0) if job.job_id in self._promoted_ids: self._promoted_ids.discard(job.job_id) @@ -1116,7 +1107,7 @@ class DownloadQueueManager: """Worker that periodically cleans up old jobs.""" while not self._shutdown_event.is_set(): try: - await asyncio.sleep(3600) # Run every hour + await asyncio.sleep(3600) self.cleanup_old_jobs() except Exception as e: log.error(f"Cleanup worker error: {e}") @@ -1130,7 +1121,6 @@ def get_download_manager() -> DownloadQueueManager: """Get the global download manager instance.""" global download_manager if download_manager is None: - # Load configuration from unshackle config from unshackle.core.config import config max_concurrent = getattr(config, "max_concurrent_downloads", 2) diff --git a/unshackle/core/api/errors.py b/unshackle/core/api/errors.py index 7fe2226d..6e891c6b 100644 --- a/unshackle/core/api/errors.py +++ b/unshackle/core/api/errors.py @@ -18,34 +18,32 @@ from aiohttp import web class APIErrorCode(str, Enum): """Standard API error codes for programmatic error handling.""" - # Client errors (4xx) - INVALID_INPUT = "INVALID_INPUT" # Missing or malformed request data - INVALID_SERVICE = "INVALID_SERVICE" # Unknown service name - INVALID_PROXY = "INVALID_PROXY" # Invalid proxy specification - INVALID_PARAMETERS = "INVALID_PARAMETERS" # Invalid download parameters + INVALID_INPUT = "INVALID_INPUT" + INVALID_SERVICE = "INVALID_SERVICE" + INVALID_PROXY = "INVALID_PROXY" + INVALID_PARAMETERS = "INVALID_PARAMETERS" - AUTH_FAILED = "AUTH_FAILED" # Authentication failure (invalid credentials/cookies) - FORBIDDEN = "FORBIDDEN" # Action not allowed - GEOFENCE = "GEOFENCE" # Content not available in region + AUTH_FAILED = "AUTH_FAILED" + FORBIDDEN = "FORBIDDEN" + GEOFENCE = "GEOFENCE" - NOT_FOUND = "NOT_FOUND" # Resource not found (title, job, etc.) - NO_CONTENT = "NO_CONTENT" # No titles/tracks/episodes found - JOB_NOT_FOUND = "JOB_NOT_FOUND" # Download job doesn't exist - SESSION_NOT_FOUND = "SESSION_NOT_FOUND" # Remote-dl session doesn't exist or expired - TRACK_NOT_FOUND = "TRACK_NOT_FOUND" # Track ID not found in session + NOT_FOUND = "NOT_FOUND" + NO_CONTENT = "NO_CONTENT" + JOB_NOT_FOUND = "JOB_NOT_FOUND" + SESSION_NOT_FOUND = "SESSION_NOT_FOUND" + TRACK_NOT_FOUND = "TRACK_NOT_FOUND" - CONFLICT = "CONFLICT" # Job/resource is in a state that disallows the action + CONFLICT = "CONFLICT" - RATE_LIMITED = "RATE_LIMITED" # Service rate limiting + RATE_LIMITED = "RATE_LIMITED" - # Server errors (5xx) - INTERNAL_ERROR = "INTERNAL_ERROR" # Unexpected server error - SERVICE_ERROR = "SERVICE_ERROR" # Streaming service API error - NETWORK_ERROR = "NETWORK_ERROR" # Network connectivity issue - DRM_ERROR = "DRM_ERROR" # DRM/license acquisition failure - DOWNLOAD_ERROR = "DOWNLOAD_ERROR" # Download process failure - SERVICE_UNAVAILABLE = "SERVICE_UNAVAILABLE" # Service temporarily unavailable - WORKER_ERROR = "WORKER_ERROR" # Download worker process error + INTERNAL_ERROR = "INTERNAL_ERROR" + SERVICE_ERROR = "SERVICE_ERROR" + NETWORK_ERROR = "NETWORK_ERROR" + DRM_ERROR = "DRM_ERROR" + DOWNLOAD_ERROR = "DOWNLOAD_ERROR" + SERVICE_UNAVAILABLE = "SERVICE_UNAVAILABLE" + WORKER_ERROR = "WORKER_ERROR" class APIError(Exception): @@ -79,32 +77,23 @@ class APIError(Exception): def default_http_status(error_code: APIErrorCode) -> int: """Map error codes to default HTTP status codes.""" status_map = { - # 400 Bad Request APIErrorCode.INVALID_INPUT: 400, APIErrorCode.INVALID_SERVICE: 400, APIErrorCode.INVALID_PROXY: 400, APIErrorCode.INVALID_PARAMETERS: 400, - # 401 Unauthorized APIErrorCode.AUTH_FAILED: 401, - # 403 Forbidden APIErrorCode.FORBIDDEN: 403, APIErrorCode.GEOFENCE: 403, - # 404 Not Found APIErrorCode.NOT_FOUND: 404, APIErrorCode.NO_CONTENT: 404, APIErrorCode.JOB_NOT_FOUND: 404, APIErrorCode.SESSION_NOT_FOUND: 404, APIErrorCode.TRACK_NOT_FOUND: 404, - # 409 Conflict APIErrorCode.CONFLICT: 409, - # 429 Too Many Requests APIErrorCode.RATE_LIMITED: 429, - # 500 Internal Server Error APIErrorCode.INTERNAL_ERROR: 500, - # 502 Bad Gateway APIErrorCode.SERVICE_ERROR: 502, APIErrorCode.DRM_ERROR: 502, - # 503 Service Unavailable APIErrorCode.NETWORK_ERROR: 503, APIErrorCode.SERVICE_UNAVAILABLE: 503, APIErrorCode.DOWNLOAD_ERROR: 500, @@ -136,7 +125,6 @@ def build_error_response( http_status = error.http_status retryable = error.retryable else: - # Generic exception - convert to INTERNAL_ERROR error_code = APIErrorCode.INTERNAL_ERROR.value message = str(error) or "An unexpected error occurred" details = {} @@ -150,25 +138,20 @@ def build_error_response( "timestamp": datetime.now(timezone.utc).isoformat(), } - # Add details if present if details: response_data["details"] = details - # Add retryable hint if specified if retryable: response_data["retryable"] = True - # Add debug information if in debug mode if debug_mode: debug_info: dict[str, Any] = { "exception_type": type(error).__name__, } - # Add traceback for debugging if isinstance(error, Exception): debug_info["traceback"] = traceback.format_exc() - # Add any extra debug info provided if extra_debug_info: debug_info.update(extra_debug_info) @@ -198,7 +181,6 @@ def categorize_exception( exc_str = str(exc).lower() exc_type = type(exc).__name__ - # Authentication errors if any(keyword in exc_str for keyword in ["auth", "login", "credential", "unauthorized", "forbidden", "token"]): return APIError( error_code=APIErrorCode.AUTH_FAILED, @@ -207,7 +189,6 @@ def categorize_exception( retryable=False, ) - # Network errors if any( keyword in exc_str for keyword in [ @@ -228,7 +209,6 @@ def categorize_exception( http_status=503, ) - # Geofence/region errors if any(keyword in exc_str for keyword in ["geofence", "region", "not available in", "territory"]): return APIError( error_code=APIErrorCode.GEOFENCE, @@ -237,7 +217,6 @@ def categorize_exception( retryable=False, ) - # Not found errors if any(keyword in exc_str for keyword in ["not found", "404", "does not exist", "invalid id"]): return APIError( error_code=APIErrorCode.NOT_FOUND, @@ -246,7 +225,6 @@ def categorize_exception( retryable=False, ) - # Rate limiting if any(keyword in exc_str for keyword in ["rate limit", "too many requests", "429", "throttle"]): return APIError( error_code=APIErrorCode.RATE_LIMITED, @@ -256,7 +234,6 @@ def categorize_exception( http_status=429, ) - # DRM errors if any(keyword in exc_str for keyword in ["drm", "license", "widevine", "playready", "decrypt"]): return APIError( error_code=APIErrorCode.DRM_ERROR, @@ -265,7 +242,6 @@ def categorize_exception( retryable=False, ) - # Service unavailable if any(keyword in exc_str for keyword in ["service unavailable", "503", "maintenance", "temporarily unavailable"]): return APIError( error_code=APIErrorCode.SERVICE_UNAVAILABLE, @@ -275,7 +251,6 @@ def categorize_exception( http_status=503, ) - # Validation errors if any(keyword in exc_str for keyword in ["invalid", "malformed", "validation"]) or exc_type in [ "ValueError", "ValidationError", @@ -287,7 +262,6 @@ def categorize_exception( retryable=False, ) - # Default to internal error for unknown exceptions return APIError( error_code=APIErrorCode.INTERNAL_ERROR, message=f"An unexpected error occurred: {exc}", diff --git a/unshackle/core/api/handlers.py b/unshackle/core/api/handlers.py index 1eb88ea6..834e2744 100644 --- a/unshackle/core/api/handlers.py +++ b/unshackle/core/api/handlers.py @@ -521,11 +521,9 @@ def serialize_drm(drm_list) -> Optional[List[Dict[str, Any]]]: exc_info=True, ) - # Get KIDs if hasattr(drm, "kids") and drm.kids: drm_info["kids"] = [str(kid) for kid in drm.kids] - # Get content keys if available if hasattr(drm, "content_keys") and drm.content_keys: drm_info["content_keys"] = {str(k): v for k, v in drm.content_keys.items()} @@ -693,12 +691,10 @@ async def search_handler(data: Dict[str, Any], request: Optional[web.Request] = details={"service": normalized_service}, ) - # Authenticate cookies = dl.get_cookie_jar(normalized_service, profile) credential = dl.get_credentials(normalized_service, profile) service_instance.authenticate(cookies, credential) - # Search results = [] try: for result in service_instance.search(): @@ -867,7 +863,6 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques log.debug(f"Episode {title.season}x{title.number} not available, skipping") continue except (Exception, SystemExit) as e: - # Handle other errors gracefully failed_episodes.append(f"S{title.season}E{title.number:02d}{part_key_suffix(title.part)}") log.debug(f"Error getting tracks for {title.season}x{title.number}: {e}") continue @@ -888,7 +883,6 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques }, ) else: - # Single episode or movie first_title = matching_titles[0] else: first_title = titles_list[0] @@ -1116,25 +1110,20 @@ async def download_handler(data: Dict[str, Any], request: Optional[web.Request] enforce_download_gates(data, request) try: - # Load service module to extract service-specific parameter defaults service_module = Services.load(normalized_service) service_specific_defaults = {} - # Extract default values from the service's click command. # Skip None defaults here: this dict overlays into job params; injecting # None for keys like `profile` would clobber serve-config overrides. # Missing required __init__ params are handled in download_manager.perform_download. if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): for param in service_module.cli.params: if hasattr(param, "name") and param.default is not None and not isinstance(param.default, enum.Enum): - # Store service-specific defaults (e.g. drm_system, hydrate_track, profile) service_specific_defaults[param.name] = param.default - # Get download manager and start workers if needed manager = get_download_manager() await manager.start_workers() - # Create download job with filtered parameters (exclude service and title_id as they're already passed) filtered_params = {k: v for k, v in data.items() if k not in ["service", "title_id"]} # Overlay any dl-relevant keys from `serve:` config (e.g. downloads, workers) so the API # respects server-side defaults without each client having to send them. @@ -1488,11 +1477,6 @@ async def prioritize_download_job_handler(job_id: str, request: Optional[web.Req ) -# --------------------------------------------------------------------------- -# Platform Handlers (profiles, config, history, maintenance) -# --------------------------------------------------------------------------- - - CONFIG_SECRET_KEY_RE = re.compile(r"secret|password|token|api_key|credential", re.IGNORECASE) @@ -1752,11 +1736,6 @@ async def env_check_handler(request: Optional[web.Request] = None) -> web.Respon return handle_api_exception(e, context={"operation": "env_check"}, debug_mode=debug_mode) -# --------------------------------------------------------------------------- -# Remote-DL Session Handlers -# --------------------------------------------------------------------------- - - SESSION_TRANSPORT_KEYS = { "service", "title_id", @@ -2005,7 +1984,6 @@ async def session_titles_handler(session_id: str, request: Optional[web.Request] titles = service_instance.get_titles() session.titles = titles - # Serialize titles and build title map if hasattr(titles, "__iter__") and not isinstance(titles, str): titles_list = list(titles) else: @@ -2196,10 +2174,8 @@ async def session_segments_handler( "drm": serialize_drm(track.drm) if hasattr(track, "drm") and track.drm else None, } - # Extract session headers/cookies for CDN access service_session = session.service_instance.session if hasattr(service_session, "headers"): - # Only include relevant headers, not all session headers headers = dict(service_session.headers) if service_session.headers else {} track_info["headers"] = headers else: @@ -2496,7 +2472,6 @@ def ensure_track_drm(track: Any) -> None: if track.drm: return - # DASH: extract from ContentProtection elements if track.data.get("dash"): from unshackle.core.manifests import DASH as DASHManifest @@ -2507,7 +2482,6 @@ def ensure_track_drm(track: Any) -> None: if track.drm: return - # HLS: fetch playlist and extract DRM from EXT-X-KEY if track.data.get("hls") and track.url: try: import m3u8 @@ -2530,7 +2504,6 @@ def ensure_track_drm(track: Any) -> None: except Exception as e: log.debug(f"HLS DRM prefetch failed for {track.id}: {e!r}") - # ISM: extract from ProtectionHeader elements if track.data.get("ism"): try: from unshackle.core.manifests import ISM as ISMManifest diff --git a/unshackle/core/api/progress.py b/unshackle/core/api/progress.py index eab489b9..ce47a9bb 100644 --- a/unshackle/core/api/progress.py +++ b/unshackle/core/api/progress.py @@ -85,7 +85,6 @@ def build_job_progress_callables( phase += f" (+{len(active) - 3} more)" else: phase = f"downloading {completed}/{total} tracks" - # segment counts and transfer speed of the track downloading now, for a granular display active_i = next((i for i in range(total) if started[i] and not done[i]), None) sink( { diff --git a/unshackle/core/api/routes.py b/unshackle/core/api/routes.py index a918cf77..12d8ee0b 100644 --- a/unshackle/core/api/routes.py +++ b/unshackle/core/api/routes.py @@ -52,7 +52,6 @@ async def cors_middleware( request: web.Request, handler: Callable[[web.Request], Awaitable[web.StreamResponse]] ) -> web.StreamResponse: """Add CORS headers to all responses.""" - # Handle preflight requests response: web.StreamResponse if request.method == "OPTIONS": response = web.Response() @@ -234,7 +233,6 @@ async def services(request: web.Request) -> web.Response: if hasattr(service_module, "TITLE_RE"): title_re = service_module.TITLE_RE - # Handle different types of TITLE_RE if isinstance(title_re, re.Pattern): service_data["title_regex"] = title_re.pattern elif isinstance(title_re, str): diff --git a/unshackle/core/api/session_store.py b/unshackle/core/api/session_store.py index 726cb0c7..634d7818 100644 --- a/unshackle/core/api/session_store.py +++ b/unshackle/core/api/session_store.py @@ -28,15 +28,15 @@ class SessionEntry: session_id: str service_tag: str - service_instance: Any # Service instance (authenticated) + service_instance: Any titles: Any = None # Titles_T from get_titles() - title_map: Dict[str, Any] = field(default_factory=dict) # title_id -> Title object - tracks: Dict[str, Track] = field(default_factory=dict) # track_id -> Track object - tracks_by_title: Dict[str, Dict[str, Track]] = field(default_factory=dict) # title_key -> {track_id -> Track} - chapters_by_title: Dict[str, List[Any]] = field(default_factory=dict) # title_key -> [Chapter] + title_map: Dict[str, Any] = field(default_factory=dict) + tracks: Dict[str, Track] = field(default_factory=dict) + tracks_by_title: Dict[str, Dict[str, Track]] = field(default_factory=dict) + chapters_by_title: Dict[str, List[Any]] = field(default_factory=dict) creator_ip: Optional[str] = None owner_key: Optional[str] = None # X-Secret-Key that owns this session - cache_tag: Optional[str] = None # per-session cache directory tag + cache_tag: Optional[str] = None input_bridge: Optional[InputBridge] = None auth_status: AuthStatus = AuthStatus.AUTHENTICATED auth_error: Optional[str] = None @@ -59,7 +59,7 @@ class SessionStore: @property def ttl(self) -> int: """Session TTL in seconds from config.""" - return config.serve.get("session_ttl", 300) # 5 min default + return config.serve.get("session_ttl", 300) @property def max_sessions(self) -> int: @@ -146,7 +146,7 @@ class SessionStore: async def loop() -> None: while True: - await asyncio.sleep(60) # Check every minute + await asyncio.sleep(60) try: await self.cleanup_expired() except Exception: @@ -199,7 +199,6 @@ class SessionStore: return len(self._sessions) -# Singleton instance session_store: Optional[SessionStore] = None diff --git a/unshackle/core/cacher.py b/unshackle/core/cacher.py index 28cee476..2f102d93 100644 --- a/unshackle/core/cacher.py +++ b/unshackle/core/cacher.py @@ -32,7 +32,6 @@ class Cacher: self.expiration = expiration if self.expiration and self.expired: - # if its expired, remove the data for safety and delete cache file self.data = None self.path.unlink() diff --git a/unshackle/core/cdm/custom_remote_cdm.py b/unshackle/core/cdm/custom_remote_cdm.py index 161c2d97..50dab66a 100644 --- a/unshackle/core/cdm/custom_remote_cdm.py +++ b/unshackle/core/cdm/custom_remote_cdm.py @@ -144,26 +144,21 @@ class CustomRemoteCDM: self.vaults = vaults self.timeout = timeout - # Device configuration device = device or {} self.device_name = device.get("name", "ChromeCDM") self.device_type_str = device.get("type", "CHROME") self.system_id = device.get("system_id", 26830) self.security_level = device.get("security_level", 3) - # Determine if this is a PlayReady CDM self._is_playready = self.device_type_str.upper() == "PLAYREADY" or ( bool(self.device_name) and self.device_name.upper().startswith("SL") ) - # Get device type enum for compatibility if self.device_type_str: self.device_type = self.get_device_type_enum(self.device_type_str) - # Authentication configuration self.auth_config = auth or {"type": "header", "header_name": "Authorization", "key": ""} - # Endpoints configuration with defaults endpoints = endpoints or {} self.endpoints = { "get_request": { @@ -198,33 +193,26 @@ class CustomRemoteCDM: }, } - # Request mapping configuration self.request_mapping = request_mapping or {} - # Response mapping configuration self.response_mapping = response_mapping or {} - # Caching configuration caching = caching or {} self.caching_enabled = caching.get("enabled", True) self.use_vaults = caching.get("use_vaults", True) and self.vaults is not None self.check_cached_first = caching.get("check_cached_first", True) - # Legacy configuration self.legacy_config = legacy or {"enabled": False} - # Session management self._sessions: Dict[bytes, Dict[str, Any]] = {} self._pssh_b64 = None self._required_kids: Optional[List[str]] = None - # HTTP session setup self._http_session = Session() self._http_session.headers.update( {"Content-Type": "application/json", "User-Agent": f"unshackle-custom-cdm/{__version__}"} ) - # Apply custom headers from auth config custom_headers = self.auth_config.get("custom_headers", {}) if custom_headers: self._http_session.headers.update(custom_headers) @@ -353,7 +341,7 @@ class CustomRemoteCDM: transform_type: Type of transformation to apply Returns: - Transformed value + Transformed value. An unknown transform_type returns the value unchanged instead of raising. Supported transforms: - base64_encode: Encode bytes/string to base64 @@ -362,7 +350,7 @@ class CustomRemoteCDM: - hex_decode: Decode hex string to bytes - json_stringify: Convert object to JSON string - json_parse: Parse JSON string to object - - parse_key_string: Parse "kid:key" format strings + - parse_key_string: Parse newline-separated "kid:key" lines, with or without a leading "--key " """ if transform_type == "base64_encode": if isinstance(value, str): @@ -399,7 +387,6 @@ class CustomRemoteCDM: return value elif transform_type == "parse_key_string": - # Handle key formats like "kid:key" or "--key kid:key" if isinstance(value, str): keys = [] for line in value.split("\n"): @@ -412,7 +399,6 @@ class CustomRemoteCDM: return keys return value - # Unknown transform type - return value unchanged return value def evaluate_condition(self, condition: str, context: Dict[str, Any]) -> bool: @@ -424,7 +410,8 @@ class CustomRemoteCDM: context: Context dictionary with values to check Returns: - True if condition is met, False otherwise + True if condition is met, False otherwise. A condition that matches none of the supported + forms is treated as unmet and returns False. Supported conditions: - "field == value": Equality check @@ -435,12 +422,10 @@ class CustomRemoteCDM: """ condition = condition.strip() - # Check for existence if " exists" in condition: field = condition.replace(" exists", "").strip() return self.get_nested_field(context, field) is not None - # Check for null comparisons if " == null" in condition: field = condition.replace(" == null", "").strip() return self.get_nested_field(context, field) is None @@ -449,7 +434,6 @@ class CustomRemoteCDM: field = condition.replace(" != null", "").strip() return self.get_nested_field(context, field) is not None - # Check for equality if " == " in condition: parts = condition.split(" == ", 1) field = parts[0].strip() @@ -457,7 +441,6 @@ class CustomRemoteCDM: actual_value = self.get_nested_field(context, field) return str(actual_value) == expected_value - # Check for inequality if " != " in condition: parts = condition.split(" != ", 1) field = parts[0].strip() @@ -465,7 +448,6 @@ class CustomRemoteCDM: actual_value = self.get_nested_field(context, field) return str(actual_value) != expected_value - # Unknown condition format - return False return False def build_request_params( @@ -490,13 +472,10 @@ class CustomRemoteCDM: 5. Nested parameter structure (create nested objects) 6. Parameter exclusions (remove unwanted params) """ - # Get mapping config for this endpoint mapping_config = self.request_mapping.get(endpoint_name, {}) - # Start with base parameters params = base_params.copy() - # 1. Apply parameter name mappings param_names = mapping_config.get("param_names", {}) if param_names: renamed_params = {} @@ -505,16 +484,13 @@ class CustomRemoteCDM: renamed_params[new_name] = params.pop(old_name) params.update(renamed_params) - # 2. Add static parameters static_params = mapping_config.get("static_params", {}) if static_params: params.update(static_params) - # 3. Add conditional parameters conditional_params = mapping_config.get("conditional_params", []) for condition_block in conditional_params: condition = condition_block.get("condition", "") - # Create context for condition evaluation context = { "device_type": self.device_type_str, "device_name": self.device_name, @@ -526,7 +502,6 @@ class CustomRemoteCDM: if self.evaluate_condition(condition, context): params.update(condition_block.get("params", {})) - # 4. Apply parameter transforms transforms = mapping_config.get("transforms", []) for transform in transforms: param_name = transform.get("param") @@ -534,7 +509,6 @@ class CustomRemoteCDM: if param_name in params: params[param_name] = self.apply_transform(params[param_name], transform_type) - # 5. Handle nested parameter structure nested_params = mapping_config.get("nested_params", {}) if nested_params: for parent_key, child_keys in nested_params.items(): @@ -545,7 +519,6 @@ class CustomRemoteCDM: if nested_obj: params[parent_key] = nested_obj - # 6. Exclude unwanted parameters exclude_params = mapping_config.get("exclude_params", []) for param_name in exclude_params: params.pop(param_name, None) @@ -601,10 +574,8 @@ class CustomRemoteCDM: This method extracts fields from the response using the response_mapping configuration, handling nested fields, type detection, and transformations. """ - # Get mapping config for this endpoint mapping_config = self.response_mapping.get(endpoint_name, {}) - # Extract fields based on mapping fields_config = mapping_config.get("fields", {}) parsed = {} @@ -613,7 +584,6 @@ class CustomRemoteCDM: if value is not None: parsed[standard_name] = value - # Apply response transforms transforms = mapping_config.get("transforms", []) for transform in transforms: field_name = transform.get("field") @@ -621,7 +591,6 @@ class CustomRemoteCDM: if field_name in parsed: parsed[field_name] = self.apply_transform(parsed[field_name], transform_type) - # Determine response type response_types = mapping_config.get("response_types", []) for response_type_config in response_types: condition = response_type_config.get("condition", "") @@ -629,14 +598,12 @@ class CustomRemoteCDM: parsed["_response_type"] = response_type_config.get("type") break - # Check success conditions success_conditions = mapping_config.get("success_conditions", []) is_success = True if success_conditions: is_success = all(self.evaluate_condition(cond, parsed) for cond in success_conditions) parsed["_is_success"] = is_success - # Extract error messages if not successful if not is_success: error_fields = mapping_config.get("error_fields", ["error", "message", "details"]) error_messages = [] @@ -675,7 +642,6 @@ class CustomRemoteCDM: if kid and key: keys.append({"kid": str(kid), "key": str(key), "type": str(key_type)}) - # Handle string format keys (e.g., "kid:key" format) elif isinstance(keys_data, str): keys = self.apply_transform(keys_data, "parse_key_string") @@ -819,7 +785,6 @@ class CustomRemoteCDM: session["pssh"] = pssh_or_wrm init_data = self.get_init_data_from_pssh(pssh_or_wrm) - # Check vaults for cached keys first if self.use_vaults and self._required_kids: vault_keys = [] for kid_str in self._required_kids: @@ -845,7 +810,6 @@ class CustomRemoteCDM: else: session["vault_keys"] = vault_keys - # Build request parameters base_params = { "scheme": self.device_name, "init_data": init_data, @@ -857,13 +821,10 @@ class CustomRemoteCDM: if session["service_certificate"]: base_params["service_certificate"] = base64.b64encode(session["service_certificate"]).decode("utf-8") - # Transform parameters based on configuration request_params = self.build_request_params("get_request", base_params, session) - # Apply authentication self.apply_authentication(self._http_session) - # Make API request endpoint_config = self.endpoints["get_request"] url = f"{self.host}{endpoint_config['path']}" timeout = endpoint_config["timeout"] @@ -873,19 +834,15 @@ class CustomRemoteCDM: if response.status_code != 200: raise requests.RequestException(f"API request failed: {response.status_code} {response.text}") - # Parse response response_data = response.json() parsed_response = self.parse_response_data("get_request", response_data) - # Check if request was successful if not parsed_response.get("_is_success", False): error_msg = parsed_response.get("_error_message", "Unknown error") raise requests.RequestException(f"API error: {error_msg}") - # Determine response type response_type = parsed_response.get("_response_type") - # Handle cached keys response if response_type == "cached_keys" or "cached_keys" in parsed_response: cached_keys = self.parse_keys_from_response("get_request", parsed_response) @@ -895,7 +852,6 @@ class CustomRemoteCDM: session["tried_cache"] = True - # Check if we have all required keys if self._required_kids: available_kids = set() for key in all_available_keys: @@ -910,20 +866,16 @@ class CustomRemoteCDM: # This allows parse_license() to properly combine cached + license keys session["cached_keys"] = cached_keys else: - # All required keys are available from cache session["keys"] = all_available_keys return b"" else: - # No required KIDs specified - return cached keys session["keys"] = all_available_keys return b"" - # Handle license request response or fetch license if keys missing challenge = parsed_response.get("challenge") remote_session_id = parsed_response.get("session_id") if challenge and remote_session_id: - # Decode challenge if it's base64 if isinstance(challenge, str): try: challenge = base64.b64decode(challenge) @@ -934,7 +886,6 @@ class CustomRemoteCDM: session["remote_session_id"] = remote_session_id return challenge - # If we have some keys but not all, return empty to skip license parsing if session.get("keys"): return b"" @@ -944,8 +895,10 @@ class CustomRemoteCDM: """ Parse license response using the custom CDM API. - This method intelligently combines cached keys with newly obtained license keys, - avoiding duplicates while ensuring all required keys are available. + Vault keys and cached keys are collected first, then any license key whose key ID is not + already present is appended. Key IDs are matched with dashes stripped and case ignored, so + an existing key wins over a license key for the same ID. If the session already holds final + keys and has no cached keys left to combine, the call is a no-op. Args: session_id: Session identifier @@ -960,16 +913,13 @@ class CustomRemoteCDM: session = self._sessions[session_id] - # Skip parsing if we already have final keys (no cached keys to combine) # If cached_keys exist (Widevine or PlayReady), we need to combine them with license keys if session["keys"] and "cached_keys" not in session: return - # Ensure we have a challenge and session ID if not session.get("challenge") or not session.get("remote_session_id"): raise ValueError("No challenge available - call get_license_challenge first") - # Prepare license message if isinstance(license_message, str): if self.is_playready and license_message.strip().startswith(" int: return min(MAX_CHUNK, max(MIN_CHUNK, content_length // 4)) -# Adaptive worker controller (opt-in): AIMD hill-climb over segment concurrency -ADAPTIVE_TICK = 4.0 # seconds between target re-evaluations -ADAPTIVE_STEP = 2 # additive increase per tick -ADAPTIVE_MIN = 2 # never drop below this many workers -ADAPTIVE_ERROR_BURST = 3 # errors within a tick that trigger multiplicative decrease -ADAPTIVE_PLATEAU_GAIN = 1.10 # min speed ratio to justify keeping an increase +ADAPTIVE_TICK = 4.0 +ADAPTIVE_STEP = 2 +ADAPTIVE_MIN = 2 +ADAPTIVE_ERROR_BURST = 3 +ADAPTIVE_PLATEAU_GAIN = 1.10 class AdaptiveWorkerController: @@ -194,12 +192,12 @@ class AdaptiveWorkerController: self.tick = tick self.window = window self._samples: deque[tuple[float, int]] = deque() - self._first_sample: Optional[float] = None # first-ever sample time (survives pruning) - self._errors = 0 # errors observed since the last tick evaluation + self._first_sample: Optional[float] = None + self._errors = 0 self._last_tick: Optional[float] = None - self._last_action: Optional[str] = None # "increase" once we have probed upward + self._last_action: Optional[str] = None self._speed_before_increase = 0.0 - self._target_before_increase = self.target # restore point for a plateau revert + self._target_before_increase = self.target self._cooldown = False def prune(self, now: float) -> None: @@ -248,7 +246,6 @@ class AdaptiveWorkerController: return self.target self._last_tick = now - # hold until half a tick of throughput history has accumulated if not self.warmed_up(now): self._errors = 0 return self.target @@ -259,7 +256,6 @@ class AdaptiveWorkerController: old = self.target if self._cooldown: - # one tick of quiet after a decrease before probing again self._cooldown = False self._last_action = None return self.target @@ -1198,9 +1194,9 @@ def download_multiprocess( p.terminate() raise RuntimeError(f"segment download child failed:\n{event['__mp_error__']}") if "total" in event: - continue # parent owns the aggregate total + continue if "downloaded" in event: - continue # child speed/status string; parent computes its own aggregate speed + continue if "advance" in event: # children report mixed granularity (segment counts, or raw bytes when a # length-1 strided chunk routes through the single-file path); the parent @@ -1256,6 +1252,10 @@ def requests( provided (e.g. from a service's get_session()), TLS fingerprinting is preserved on all segment downloads. + A file already at its computed save path counts as finished from an earlier run. It is reported as + downloaded at its on-disk size without any request, so calling this again resumes a batch from the + segments the earlier run completed. + Yields the following download status updates while chunks are downloading: - {total: 123} (there are 123 chunks to download) @@ -1360,7 +1360,7 @@ def requests( spec = build_session_spec(session) if spec is not None: yield from download_multiprocess( - urls=cast("list[Any]", urls), # normalized to a list above + urls=cast("list[Any]", urls), output_dir=output_dir, filename=filename, headers=headers, @@ -1471,8 +1471,6 @@ def requests( yield dict(downloaded="[yellow]CANCELLED") raise else: - # Segmented download with thread pool - # Speed is tracked here on the main thread, not in workers total_bytes = 0 start_time = time.time() last_speed_report = start_time @@ -1518,7 +1516,7 @@ def requests( def download_worker(index: int, url_item: dict[str, Any], hedge: bool = False) -> None: item = dict(url_item) save_path = item.pop("save_path") - if save_path.exists(): # finished in a previous run + if save_path.exists(): with seg_lock: if index in seg_done: return @@ -1664,7 +1662,7 @@ def requests( continue item = dict(url_item) save_path = item.pop("save_path") - if save_path.exists(): # finished in a previous run; let the normal fast-path handle it + if save_path.exists(): tail_skipped.add(index) continue if has_range_header(item): # byte-range slice; part-mode would clobber its Range @@ -1724,38 +1722,32 @@ def requests( try: while pending or remaining: - # Drain queued events — batch advances, track bytes for speed while True: try: event = event_queue.get_nowait() except Empty: break - # Accumulate advance events for batched yield advance = event.get("advance") if advance: pending_advance += advance continue - # Track bytes from completed segments for speed calculation written = event.get("written") if written: total_bytes += written if controller: controller.record_bytes(written, time.time()) - # Pass through other events (file_downloaded, total, etc.) yield event - # Yield batched advances every drain cycle for responsive progress bar if pending_advance > 0: yield dict(advance=pending_advance) pending_advance = 0 now = time.time() - # Adaptive: re-evaluate the worker target and top up in-flight futures target = max_workers if controller: target = controller.update(now, len(pending) + len(remaining)) - maybe_tail_boost(target) # spend spare workers on the tail before topping up + maybe_tail_boost(target) if len(pending) < target: # In the final stride (unstarted segments <= one worker target), hold back # boost candidates so idle workers accumulate for a range-split; only @@ -1768,7 +1760,6 @@ def requests( elif len(pending) < queue_depth: submit(queue_depth - len(pending)) - # Yield speed every 0.5s (throttled to avoid spamming Rich) if now - last_speed_report > 0.5: if controller: rolling = controller.speed(now) @@ -1813,7 +1804,6 @@ def requests( if len(seg_done) == len(urls): break - # Wait efficiently for next future completion (OS condition variable) completed, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED) for future in completed: exc = future.exception() @@ -1860,7 +1850,6 @@ def requests( force_unblock_stream(stream) pool.shutdown(wait=not DOWNLOAD_CANCELLED.is_set(), cancel_futures=True) - # Drain remaining events while True: try: event = event_queue.get_nowait() @@ -1875,7 +1864,6 @@ def requests( total_bytes += written yield event - # Flush remaining advances and final speed if pending_advance > 0: yield dict(advance=pending_advance) elapsed = time.time() - start_time diff --git a/unshackle/core/drm/monalisa.py b/unshackle/core/drm/monalisa.py index fa4c2182..50b06328 100644 --- a/unshackle/core/drm/monalisa.py +++ b/unshackle/core/drm/monalisa.py @@ -72,7 +72,6 @@ class MonaLisa: self._ticket = ticket - # Store AES key for second-stage decryption if isinstance(aes_key, str): self._aes_key = bytes.fromhex(aes_key) else: @@ -83,7 +82,6 @@ class MonaLisa: self._key: Optional[str] = None self.data: dict = kwargs or {} - # Extract keys immediately self.extract_keys() def extract_keys(self) -> None: @@ -171,7 +169,6 @@ class MonaLisa: import base64 try: - # Decode base64 PSSH to get raw bytes if isinstance(self._ticket, bytes): data = self._ticket else: @@ -180,7 +177,6 @@ class MonaLisa: # Content ID is at bytes 21-75 (55 bytes) if len(data) >= 76: content_id = data[21:76].decode("ascii") - # Validate it looks like a content ID if content_id.startswith("H5DCID-"): return content_id except (ValueError, TypeError): @@ -233,7 +229,6 @@ class MonaLisa: else: raise MonaLisa.Exceptions.DecryptionFailed(f"Segment file does not exist: {segment_path}") - # Stage 1: ML-Worker decryption cmd = [str(worker_path), str(self._key), str(bbts_path), str(ents_path)] startupinfo = None @@ -263,14 +258,12 @@ class MonaLisa: f"Decrypted .ents file was not created for {segment_path.name}" ) - # Stage 2: AES-ECB decryption with open(ents_path, "rb") as f: ents_data = f.read() crypto = AES.new(self._aes_key, AES.MODE_ECB) decrypted_data = unpad(crypto.decrypt(ents_data), AES.block_size) - # Write decrypted segment back to original path with open(segment_path, "wb") as f: f.write(decrypted_data) diff --git a/unshackle/core/drm/playready.py b/unshackle/core/drm/playready.py index 18434016..e0858dc2 100644 --- a/unshackle/core/drm/playready.py +++ b/unshackle/core/drm/playready.py @@ -89,7 +89,6 @@ class PlayReady: pssh_bytes = base64.b64decode(pssh_b64) # Try to find XML in the PSSH data - # PlayReady PSSH usually has XML embedded in it pssh_str = pssh_bytes.decode("utf-16le", errors="ignore") # Find WRMHEADER @@ -109,7 +108,6 @@ class PlayReady: kids = [] - # Extract from CUSTOMATTRIBUTES/KIDS kid_elements = root.findall(".//pr:CUSTOMATTRIBUTES/pr:KIDS/pr:KID", ns) for kid_elem in kid_elements: value = kid_elem.get("VALUE") @@ -439,7 +437,6 @@ class PlayReady: output_path = path.with_stem(f"{path.stem}_decrypted") - # Build key arguments key_args = [] for kid, key in self.content_keys.items(): kid_hex = kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "") diff --git a/unshackle/core/drm/widevine.py b/unshackle/core/drm/widevine.py index d44f4e49..2f72ad76 100644 --- a/unshackle/core/drm/widevine.py +++ b/unshackle/core/drm/widevine.py @@ -353,7 +353,6 @@ class Widevine: output_path = path.with_stem(f"{path.stem}_decrypted") - # Build key arguments key_args = [] for kid, key in self.content_keys.items(): kid_hex = kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "") @@ -456,7 +455,6 @@ class Widevine: if ":ERROR:" in line: had_error = True if "Insufficient bits in bitstream for given AVC profile" in line: - # this is a warning and is something we don't have to worry about continue shaka_log_buffer += f"{line.strip()}\n" diff --git a/unshackle/core/events.py b/unshackle/core/events.py index 79316eab..d5622b96 100644 --- a/unshackle/core/events.py +++ b/unshackle/core/events.py @@ -8,13 +8,9 @@ from typing import Any, Callable class Events: class Types(Enum): _reserved = 0 - # A Track's segment has finished downloading SEGMENT_DOWNLOADED = 1 - # Track has finished downloading TRACK_DOWNLOADED = 2 - # Track has finished decrypting TRACK_DECRYPTED = 3 - # Track has finished repacking TRACK_REPACKED = 4 # Track is about to be Multiplexed into a Container TRACK_MULTIPLEX = 5 diff --git a/unshackle/core/manifests/dash.py b/unshackle/core/manifests/dash.py index 206155f2..758e5259 100644 --- a/unshackle/core/manifests/dash.py +++ b/unshackle/core/manifests/dash.py @@ -105,6 +105,10 @@ class DASH: track language value if the manifest does not list language information. period_filter: Filter out period's within the manifest. + Only the first main-content period becomes tracks. Later periods do not add tracks of their + own, but download_track stitches the segments of every content period into each returned + track. + All Track URLs will be a list of segment URLs. """ tracks = Tracks() @@ -253,7 +257,6 @@ class DASH: ) ) - # only get tracks from the first main-content period break tracks.manifest_url = self.url @@ -338,10 +341,9 @@ class DASH: if kid not in drm_obj.content_keys: drm_obj.content_keys[kid] = key - # Collect segments from all content periods in the manifest all_periods = manifest.findall("Period") segments: list[tuple[str, Optional[str]]] = [] - seen_segments: set[tuple[str, Optional[str]]] = set() # dedupe segments across periods in O(1) + seen_segments: set[tuple[str, Optional[str]]] = set() segment_durations: list[int] = [] segment_timescale: float = 0 init_data: Optional[bytes] = None @@ -526,7 +528,6 @@ class DASH: with open(save_path, "r+b") as collapsed: collapsed.truncate(int(segments[-1][1].split("-")[1]) + 1) else: - # Verify output directory exists and contains files if not save_dir.exists(): error_msg = f"Output directory does not exist: {save_dir}" log_event( @@ -558,14 +559,13 @@ class DASH: "save_dir": str(save_dir), "save_dir_exists": save_dir.exists(), "segments_found": len(segments_to_merge), - "segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10 + "segment_files": [f.name for f in segments_to_merge[:10]], "downloader": "requests", }, ) if not segments_to_merge: error_msg = f"No segment files found in output directory: {save_dir}" - # List all contents of the directory for debugging all_contents = list(save_dir.iterdir()) if save_dir.exists() else [] log_event( "manifest_dash_download_no_segments", @@ -618,7 +618,6 @@ class DASH: events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=None) progress(downloaded="Decrypted", completed=100, total=100) - # Clean up empty segment directory if save_dir.exists() and save_dir.name.endswith("_segments"): try: save_dir.rmdir() @@ -1235,9 +1234,9 @@ class DASH: if d[0:2] != "PT" and not has_ymd: raise ValueError("Input data is not a valid time string.") if has_ymd: - d = d[6:].upper() # skip `P0Y0M0DT` + d = d[6:].upper() else: - d = d[2:].upper() # skip `PT` + d = d[2:].upper() m = re.findall(r"([\d.]+.)", d) return sum(float(x[0:-1]) * {"H": 60 * 60, "M": 60, "S": 1}[x[-1].upper()] for x in m) diff --git a/unshackle/core/manifests/hls.py b/unshackle/core/manifests/hls.py index 0117f40c..9a831ed8 100644 --- a/unshackle/core/manifests/hls.py +++ b/unshackle/core/manifests/hls.py @@ -76,7 +76,6 @@ class HLS: res = session.get(url, **args) - # Handle requests and rnet response objects if isinstance(res, requests.Response): if not res.ok: raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res) @@ -227,7 +226,6 @@ class HLS: descriptor=Video.Descriptor.HLS, drm=session_drm, data={"hls": {"playlist": playlist}}, - # video track args **( dict( range_=video_range, @@ -287,7 +285,6 @@ class HLS: descriptor=Audio.Descriptor.HLS, drm=session_drm if media.type == "AUDIO" else None, data={"hls": {"media": media}}, - # audio track args **( dict( bitrate=0, # TODO: M3U doesn't seem to state bitrate? @@ -390,9 +387,9 @@ class HLS: val = self.ue() return (val + 1) // 2 if val & 1 else -(val // 2) - # Find SPS NAL unit via start code # H.264: NAL type 7 (SPS), identified by byte & 0x1F == 7 # H.265: NAL type 33 (SPS), identified by (byte >> 1) & 0x3F == 33 + # Find SPS NAL unit via start code for i in range(len(data) - 4): start3 = data[i : i + 3] == b"\x00\x00\x01" start4 = data[i : i + 4] == b"\x00\x00\x00\x01" @@ -532,7 +529,6 @@ class HLS: cdm = ctx.cdm if proxy: - # Handle proxies differently based on session type if isinstance(session, Session): session.proxies.update({"all": proxy}) @@ -541,7 +537,6 @@ class HLS: if track.from_file: master = m3u8.load(str(track.from_file)) else: - # Get the playlist text and handle both session types response = session.get(track.url) if isinstance(response, requests.Response) or isinstance(response, RnetResponse): if not response.ok: @@ -595,14 +590,11 @@ class HLS: raise elif isinstance(media_drm, ClearKey): # AES-128 (ClearKey) needs no license server - the key is already fetched. - # Without this branch session_drm stayed None and segments were never - # decrypted (they were merged still-encrypted, producing a broken file). track.drm = [media_drm] session_drm = media_drm initial_drm_key = media_playlist_key initial_drm_licensed = True - # Fall back to session DRM if media playlist has no matching keys if not initial_drm_licensed and session_drm and isinstance(session_drm, (Widevine, PlayReady)): try: if not license_widevine: @@ -817,14 +809,13 @@ class HLS: raise ValueError(f"Missing {range_len - len(files)} segment files for {segment_range}...") if isinstance(drm, (Widevine, PlayReady)): - # with widevine we can merge all segments and decrypt once merge(to=merged_path, via=files, delete=True, include_map_data=True) drm.decrypt(merged_path) assert_fragments_decrypted(merged_path) merged_path.rename(decrypted_path) else: - # with other drm we must decrypt separately and then merge them - # for aes this is because each segment likely has 16-byte padding + # AES segments must be decrypted individually before merging: each segment + # likely carries its own 16-byte padding key_obj = encryption_data[0] # AES-128 with no explicit IV: each segment's IV is its media sequence # number (RFC 8216 §5.2). Without this the engine used a zero IV and the @@ -904,13 +895,11 @@ class HLS: else: init_range_header = {} - # Handle both session types for init section request res = session.get( url=urljoin(segment.init_section.base_uri, segment.init_section.uri), headers=init_range_header, ) - # Check response based on session type if isinstance(res, requests.Response) or isinstance(res, RnetResponse): res.raise_for_status() init_content = res.content @@ -982,11 +971,9 @@ class HLS: """Find all segment files recursively in any directory structure created by downloaders.""" segments = [] - # First check direct files in the directory if directory.exists(): segments.extend([x for x in directory.iterdir() if x.is_file()]) - # If no direct files, recursively search subdirectories if not segments: for subdir in directory.iterdir(): if subdir.is_dir(): @@ -994,7 +981,6 @@ class HLS: return sorted(segments) - # finally merge all the discontinuity save files together to the final path segments_to_merge = find_segments_recursively(save_dir) log_event( @@ -1007,7 +993,7 @@ class HLS: "save_dir": str(save_dir), "save_dir_exists": save_dir.exists(), "segments_found": len(segments_to_merge), - "segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10 + "segment_files": [f.name for f in segments_to_merge[:10]], "downloader": "requests", }, ) @@ -1045,12 +1031,10 @@ class HLS: os.fsync(f.fileno()) discontinuity_file.unlink() - # Clean up empty segment directory if save_dir.exists() and save_dir.name.endswith("_segments"): try: save_dir.rmdir() except OSError: - # Directory might not be empty, try removing recursively shutil.rmtree(save_dir, ignore_errors=True) progress(downloaded="Downloaded", completed=100, total=100) @@ -1069,10 +1053,8 @@ class HLS: Returns the file size of the merged file. """ - # Track segment directories for cleanup segment_dirs = set() for segment in segments: - # Track all parent directories that contain segments current_dir = segment.parent while current_dir.name and "_segments" in str(current_dir): segment_dirs.add(current_dir) @@ -1089,7 +1071,6 @@ class HLS: except OSError: pass # Directory cleanup failed, but merge succeeded - # Try FFmpeg concat first (preferred method) if binaries.FFMPEG: try: demuxer_file = save_path.parent / f"ffmpeg_concat_demuxer_{save_path.stem}.txt" @@ -1130,13 +1111,10 @@ class HLS: return save_path.stat().st_size except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError) as e: - # FFmpeg failed, clean up demuxer file and fall back to binary concat logging.getLogger("HLS").debug(f"FFmpeg concat failed ({e}), falling back to binary concatenation") demuxer_file.unlink(missing_ok=True) - # Remove partial output file if it exists save_path.unlink(missing_ok=True) - # Fallback: Binary concatenation logging.getLogger("HLS").debug(f"Using binary concatenation for {len(segments)} segments") with open(save_path, "wb") as output_file: for segment in segments: diff --git a/unshackle/core/manifests/ism.py b/unshackle/core/manifests/ism.py index 03e85ef0..c8c8cade 100644 --- a/unshackle/core/manifests/ism.py +++ b/unshackle/core/manifests/ism.py @@ -477,7 +477,6 @@ class ISM: status_update["downloaded"] = f"ISM {downloaded}" progress(**status_update) - # Verify output directory exists and contains files if not save_dir.exists(): error_msg = f"Output directory does not exist: {save_dir}" log_event( @@ -509,7 +508,7 @@ class ISM: "save_dir": str(save_dir), "save_dir_exists": save_dir.exists(), "segments_found": len(segments_to_merge), - "segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10 + "segment_files": [f.name for f in segments_to_merge[:10]], "downloader": "requests", }, ) diff --git a/unshackle/core/manifests/ism_init.py b/unshackle/core/manifests/ism_init.py index 02bb1ac9..fd8e8df9 100644 --- a/unshackle/core/manifests/ism_init.py +++ b/unshackle/core/manifests/ism_init.py @@ -20,7 +20,6 @@ import binascii import struct from typing import Iterator, Optional -# Big-endian field packers (named for the bit widths they encode). u8 = struct.Struct(">B") u16 = struct.Struct(">H") u32 = struct.Struct(">I") @@ -31,8 +30,7 @@ s1616 = struct.Struct(">hxx") # 16.16 fixed-point u1616 = struct.Struct(">Hxx") s32 = struct.Struct(">i") -# 3x3 transformation matrix (identity), as stored in tkhd/mvhd. -# Exactly 9 int32s; extras shift every field after it and desync pymp4. +# The tkhd/mvhd matrix is exactly 9 int32s; extras shift every field after it and desync pymp4. UNITY_MATRIX = ( s32.pack(0x10000) + s32.pack(0) * 2 @@ -62,7 +60,6 @@ PIFF_SENC_UUID = bytes.fromhex("A2394F525A9B4F14A2446C427C648DF4") TTML_NAMESPACE = b"http://www.w3.org/ns/ttml\0" -# ISO/IEC 14496-3 samplingFrequencyIndex table for AudioSpecificConfig. AAC_SAMPLING_FREQUENCY_INDEX = { 96000: 0x0, 88200: 0x1, @@ -639,7 +636,7 @@ def synthesize_aac_codec_private_data(fourcc: str, sampling_rate: int, channels: (0x05 << 3) | (freq >> 1), ((freq & 0x01) << 7) | (channels << 3) | (ext_freq >> 1), ((ext_freq & 0x01) << 7) | (0x02 << 2), # core object type = AAC LC - 0x00, # alignment bits + 0x00, ) ) return bytes(((0x02 << 3) | (freq >> 1), ((freq & 0x01) << 7) | (channels << 3))) @@ -715,11 +712,9 @@ def build_init_segment( if len(lang) != 3 or not all("a" <= c <= "z" for c in lang): lang = "und" - # --- ftyp --- ftyp = box(b"ftyp", b"isml" + u32.pack(1) + b"iso5" + b"iso6" + b"piff" + b"msdh") - # --- mvhd --- - # version 1: at the 10 MHz ISM timescale a 32-bit duration overflows after ~429s + # version 1: at the 10 MHz ISM timescale a 32-bit duration overflows after ~429s. mvhd = full_box( b"mvhd", 1, @@ -737,7 +732,6 @@ def build_init_segment( + u32.pack(0xFFFFFFFF), ) - # --- tkhd --- tkhd = full_box( b"tkhd", 1, @@ -757,7 +751,6 @@ def build_init_segment( + u1616.pack(height), ) - # --- mdhd + hdlr --- packed_lang = ((ord(lang[0]) - 0x60) << 10) | ((ord(lang[1]) - 0x60) << 5) | (ord(lang[2]) - 0x60) mdhd = full_box( b"mdhd", @@ -780,11 +773,9 @@ def build_init_segment( hdlr = full_box(b"hdlr", 0, 0, u32.pack(0) + b"vide" + u32.pack(0) * 3 + b"VideoHandler\0") media_header = full_box(b"vmhd", 0, 1, u16.pack(0) + u16.pack(0) * 3) - # --- dinf --- dref = full_box(b"dref", 0, 0, u32.pack(1) + full_box(b"url ", 0, SELF_CONTAINED, b"")) dinf = box(b"dinf", dref) - # --- stsd sample entry --- sample_entry_payload = u8.pack(0) * 6 + u16.pack(1) # reserved + data reference index if stream_type == "video": sample_entry_payload += ( @@ -850,7 +841,7 @@ def build_init_segment( sample_entry_box = box(b"enca", sample_entry_payload) else: sample_entry_box = box(codec_fourcc, sample_entry_payload) - else: # text + else: if fourcc in ("TTML", "STPP", "DFXP"): # XMLSubtitleSampleEntry: namespace + schema_location + aux mime types. sample_entry_payload += TTML_NAMESPACE + b"\0" + b"\0" @@ -864,7 +855,6 @@ def build_init_segment( # second config to build. The server's larger second entry only repeats its VPS. stsd = full_box(b"stsd", 0, 0, u32.pack(2) + sample_entry_box * 2) - # --- empty sample tables (fragmented: real samples live in moof/traf) --- stbl = box( b"stbl", stsd @@ -878,7 +868,6 @@ def build_init_segment( mdia = box(b"mdia", mdhd + hdlr + minf) trak = box(b"trak", tkhd + mdia) - # --- mvex (mehd + trex) signals a fragmented file --- mehd = full_box(b"mehd", 1, 0, u64.pack(duration)) trex = full_box( b"trex", diff --git a/unshackle/core/music/display.py b/unshackle/core/music/display.py index 19ceac8c..60aa7945 100644 --- a/unshackle/core/music/display.py +++ b/unshackle/core/music/display.py @@ -13,7 +13,6 @@ from dataclasses import dataclass from io import BytesIO from typing import Any, Optional -# RenderableType is rich's union of "things that can be printed to a console". from rich import box from rich.console import Group, RenderableType from rich.padding import Padding @@ -154,7 +153,6 @@ def render_album_header( artwork: Optional[RenderableType] = None, ) -> Optional[RenderableType]: """Render the release metadata grid, optionally placing artwork beside it.""" - # Table.grid is a borderless table used purely for column alignment. grid = Table.grid(padding=(0, 2)) grid.add_column(style="orchid1", no_wrap=True) grid.add_column() diff --git a/unshackle/core/providers/__init__.py b/unshackle/core/providers/__init__.py index 9c4160cf..61835a52 100644 --- a/unshackle/core/providers/__init__.py +++ b/unshackle/core/providers/__init__.py @@ -63,9 +63,6 @@ def get_provider(name: str) -> Optional[MetadataProvider]: return p if p.is_available() else None -# -- Public API (replaces tags.py functions) -- - - def search_metadata( title: str, year: Optional[int], @@ -87,7 +84,6 @@ def search_metadata( ordered = provider_order(kind, anime) - # Check cache first if title_cacher and cache_title_id: for cls in ordered: p = cls() @@ -100,7 +96,6 @@ def search_metadata( log.debug("Using cached %s data for %r", p.NAME, title) return result - # Search providers in priority order for cls in ordered: p = cls() if not p.is_available(): @@ -111,9 +106,7 @@ def search_metadata( log.debug("%s search failed: %s", p.NAME, exc) continue if result and result.title and fuzzy_match(result.title, title): - # Enrich with cross-referenced IDs if we have IMDB but missing TMDB/TVDB enrich_ids(result) - # Cache the result (include enriched IDs so they survive round-trip) if title_cacher and cache_title_id and result.raw: try: cache_data = result.raw @@ -141,7 +134,6 @@ def get_title_by_id( cache_account_hash: Optional[str] = None, ) -> Optional[str]: """Get title name by TMDB ID.""" - # Check cache first if title_cacher and cache_title_id: cached = title_cacher.get_cached_provider("tmdb", cache_title_id, kind, cache_region, cache_account_hash) if cached and cached.get("detail"): @@ -158,7 +150,6 @@ def get_title_by_id( if not result: return None - # Cache if possible if title_cacher and cache_title_id and result.raw: try: ext_ids = tmdb.get_external_ids(tmdb_id, kind) @@ -185,7 +176,6 @@ def get_year_by_id( cache_account_hash: Optional[str] = None, ) -> Optional[int]: """Get release year by TMDB ID.""" - # Check cache first if title_cacher and cache_title_id: cached = title_cacher.get_cached_provider("tmdb", cache_title_id, kind, cache_region, cache_account_hash) if cached and cached.get("detail"): @@ -203,7 +193,6 @@ def get_year_by_id( if not result: return None - # Cache if possible if title_cacher and cache_title_id and result.raw: try: ext_ids = tmdb.get_external_ids(tmdb_id, kind) @@ -253,7 +242,6 @@ def fetch_external_ids( cache_account_hash: Optional[str] = None, ) -> ExternalIds: """Get external IDs by TMDB ID.""" - # Check cache first if title_cacher and cache_title_id: cached = title_cacher.get_cached_provider("tmdb", cache_title_id, kind, cache_region, cache_account_hash) if cached and cached.get("external_ids"): @@ -271,7 +259,6 @@ def fetch_external_ids( return ExternalIds() ext = tmdb.get_external_ids(tmdb_id, kind) - # Cache if possible if title_cacher and cache_title_id: try: detail = None @@ -373,9 +360,6 @@ def resolve_by_ids( return result -# -- Internal helpers -- - - # trust ranking for cross-validating enrichments; `metadata_providers` filters this # set but sets search order only, not the ranking ENRICHMENT_AUTHORITY: tuple[str, ...] = ("tmdb", "simkl", "tvdb") @@ -402,7 +386,6 @@ def enrich_ids(result: MetadataResult) -> None: kind = result.kind or "movie" - # Step 1: Collect enrichment results from all available providers authority = {name: i for i, name in enumerate(enrichment_providers(kind))} enrichments: list[tuple[str, ExternalIds]] = [] for provider_name in authority: @@ -420,10 +403,8 @@ def enrich_ids(result: MetadataResult) -> None: if not enrichments: return - # Step 2: Cross-validate using tmdb_id as anchor — drop providers that disagree validated = validate_enrichments(enrichments, authority) - # Step 3: Merge validated data (fill gaps only) for _provider_name, ext in validated: if not ids.tmdb_id and ext.tmdb_id: ids.tmdb_id = ext.tmdb_id @@ -444,7 +425,6 @@ def validate_enrichments( """ from collections import Counter - # Collect tmdb_id votes tmdb_votes: dict[str, int] = {} for provider_name, ext in enrichments: if ext.tmdb_id is not None: @@ -453,7 +433,6 @@ def validate_enrichments( if len(set(tmdb_votes.values())) <= 1: return enrichments # all agree or only one voted — no conflict - # Find the authoritative tmdb_id value_counts = Counter(tmdb_votes.values()) most_common_val, most_common_count = value_counts.most_common(1)[0] @@ -467,7 +446,6 @@ def validate_enrichments( ) anchor_tmdb_id = tmdb_votes[best_provider] - # Drop any provider that disagrees validated: list[tuple[str, ExternalIds]] = [] for provider_name, ext in enrichments: if ext.tmdb_id is not None and ext.tmdb_id != anchor_tmdb_id: diff --git a/unshackle/core/providers/_base.py b/unshackle/core/providers/_base.py index d2a29689..6a89f194 100644 --- a/unshackle/core/providers/_base.py +++ b/unshackle/core/providers/_base.py @@ -38,8 +38,8 @@ class MetadataResult: kind: Optional[str] = None # "movie" or "tv" external_ids: ExternalIds = field(default_factory=ExternalIds) original_language: Optional[str] = None # alpha-2 or alpha-3, whichever the provider speaks - source: str = "" # provider name, e.g. "tmdb", "simkl", "imdb" - raw: Optional[dict] = None # original API response for caching + source: str = "" + raw: Optional[dict] = None class MetadataProvider(metaclass=ABCMeta): diff --git a/unshackle/core/providers/omdb.py b/unshackle/core/providers/omdb.py index 083bfe93..032f7ff1 100644 --- a/unshackle/core/providers/omdb.py +++ b/unshackle/core/providers/omdb.py @@ -9,7 +9,6 @@ from langcodes import Language from unshackle.core.config import config from unshackle.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, clean, fuzzy_match -# Mapping from our kind ("movie"/"tv") to OMDb title types KIND_TO_TYPE: dict[str, str] = { "movie": "movie", "tv": "series", diff --git a/unshackle/core/providers/simkl.py b/unshackle/core/providers/simkl.py index c61c174f..c6d9cdb7 100644 --- a/unshackle/core/providers/simkl.py +++ b/unshackle/core/providers/simkl.py @@ -21,7 +21,6 @@ class SimklProvider(MetadataProvider): def search(self, title: str, year: Optional[int], kind: str) -> Optional[MetadataResult]: self.log.debug("Searching Simkl for %r (%s, %s)", title, kind, year) - # Construct appropriate filename based on type filename = f"{title}" if year: filename = f"{title} {year}" @@ -42,7 +41,6 @@ class SimklProvider(MetadataProvider): self.log.debug("Simkl search failed: %s", exc) return None - # Handle case where SIMKL returns empty list (no results) if isinstance(data, list): self.log.debug("Simkl returned list (no matches) for %r", filename) return None @@ -77,11 +75,9 @@ class SimklProvider(MetadataProvider): if not simkl_id: return None - # Map SIMKL type to endpoint simkl_type = entry.get("type", "") endpoint = "tv" if simkl_type in ("tv", "anime") else "movies" - # Fetch full details to get cross-referenced IDs try: r2 = self.session.get( f"{self.BASE_URL}/{endpoint}/{simkl_id}", diff --git a/unshackle/core/providers/tmdb.py b/unshackle/core/providers/tmdb.py index 60014161..8d33ad4e 100644 --- a/unshackle/core/providers/tmdb.py +++ b/unshackle/core/providers/tmdb.py @@ -72,7 +72,6 @@ class TMDBProvider(MetadataProvider): if best_id is None: return None - # Fetch full detail for caching detail = self.fetch_detail(best_id, kind) ext_raw = self.fetch_external_ids_raw(best_id, kind) @@ -127,7 +126,12 @@ class TMDBProvider(MetadataProvider): ) def find_by_imdb_id(self, imdb_id: str, kind: str) -> Optional[ExternalIds]: - """Look up TMDB/TVDB IDs from an IMDB ID using TMDB's /find endpoint.""" + """Look up TMDB/TVDB IDs from an IMDB ID using TMDB's /find endpoint. + + When the requested kind has no results, the other kind is tried, so the returned + tmdb_kind can differ from the kind asked for. Returns None if the request fails or + nothing matches. + """ self.log.debug("Looking up IMDB ID %s on TMDB", imdb_id) try: r = self.session.get( @@ -141,14 +145,12 @@ class TMDBProvider(MetadataProvider): self.log.debug("TMDB find by IMDB ID failed: %s", exc) return None - # Check movie_results or tv_results based on kind if kind == "movie": results = data.get("movie_results") or [] else: results = data.get("tv_results") or [] if not results: - # Try the other type as fallback fallback_key = "tv_results" if kind == "movie" else "movie_results" results = data.get(fallback_key) or [] if results: @@ -165,7 +167,6 @@ class TMDBProvider(MetadataProvider): self.log.debug("TMDB find -> ID %s (%s) for IMDB %s", tmdb_id, kind, imdb_id) - # Now fetch the full external IDs from TMDB to get TVDB etc. ext_raw = self.fetch_external_ids_raw(tmdb_id, kind) return ExternalIds( diff --git a/unshackle/core/proxies/gluetun.py b/unshackle/core/proxies/gluetun.py index 59972186..bea26059 100644 --- a/unshackle/core/proxies/gluetun.py +++ b/unshackle/core/proxies/gluetun.py @@ -16,7 +16,6 @@ from unshackle.core.proxies.proxy import Proxy from unshackle.core.utilities import get_country_code, get_country_name, get_debug_logger, log_event from unshackle.core.utils.ip_info import get_ip_info -# Global registry for cleanup on exit gluetun_instances: list["Gluetun"] = [] cleanup_lock = threading.Lock() cleanup_registered = False @@ -88,7 +87,6 @@ class Gluetun(Proxy): --proxy gluetun:nordvpn:de """ - # Mapping of common VPN provider names to Gluetun identifiers PROVIDER_MAPPING = { "windscribe": "windscribe", "expressvpn": "expressvpn", @@ -108,7 +106,6 @@ class Gluetun(Proxy): # Windscribe uses specific region names instead of country codes # See: https://github.com/qdm12/gluetun-wiki/blob/main/setup/providers/windscribe.md WINDSCRIBE_REGION_MAP = { - # Country codes to Windscribe region names "us": "US East", "us-east": "US East", "us-west": "US West", @@ -192,7 +189,6 @@ class Gluetun(Proxy): auth_password: Optional HTTP proxy authentication password verify_ip: Automatically verify IP and region after connection (default: True) """ - # Check Docker availability using binaries module if not binaries.Docker: raise RuntimeError( "Docker is not available. Please install Docker to use Gluetun proxy.\n" @@ -210,19 +206,15 @@ class Gluetun(Proxy): # Track active containers: {query_key: {"container_name": ..., "port": ..., ...}} self.active_containers = {} - # Lock for thread-safe port allocation self._port_lock = threading.Lock() - # Validate provider configurations for provider_name, config in self.providers.items(): self.validate_provider_config(provider_name, config) - # Register this instance for cleanup on exit register_cleanup() with cleanup_lock: gluetun_instances.append(self) - # Log initialization log_event( "gluetun_init", level="INFO", @@ -250,7 +242,6 @@ class Gluetun(Proxy): Returns: HTTP proxy URI or None if unavailable """ - # Parse query parts = query.split(":") if len(parts) != 2: raise ValueError(f"Invalid query format: '{query}'. Expected 'provider:region' (e.g., 'windscribe:us')") @@ -258,17 +249,14 @@ class Gluetun(Proxy): provider_name = parts[0].lower() region = parts[1].lower() - # Check if provider is configured if provider_name not in self.providers: available = ", ".join(self.providers.keys()) raise ValueError(f"VPN provider '{provider_name}' not configured. Available providers: {available}") - # Create query key for tracking query_key = f"{provider_name}:{region}" container_name = f"{self.container_prefix}-{provider_name}-{region}" - # Check if container already exists (in memory OR in Docker) - # This handles multiple concurrent Unshackle sessions + # Handle multiple concurrent Unshackle sessions: check if container exists in memory or in Docker if query_key in self.active_containers: container = self.active_containers[query_key] if self.is_container_running(container["container_name"]): @@ -282,7 +270,6 @@ class Gluetun(Proxy): "port": container["port"], }, ) - # Re-verify if needed if self.verify_ip: self.verify_container(query_key) return self.build_proxy_uri(container["port"]) @@ -302,15 +289,12 @@ class Gluetun(Proxy): "port": existing_info["port"], }, ) - # Re-verify if needed if self.verify_ip: self.verify_container(query_key) return self.build_proxy_uri(existing_info["port"]) - # Get provider configuration provider_config = self.providers[provider_name] - # Determine server location server_countries = provider_config.get("server_countries", {}) server_cities = provider_config.get("server_cities", {}) server_hostnames = provider_config.get("server_hostnames", {}) @@ -319,23 +303,18 @@ class Gluetun(Proxy): city = server_cities.get(region) hostname = server_hostnames.get(region) - # Check if region is a specific server pattern (e.g., us1239, uk5678) - # Format: 2-letter country code + number + # Region may specify a server: country code + number (e.g., us1239, uk5678) specific_server_match = re.match(r"^([a-z]{2})(\d+)$", region, re.IGNORECASE) if specific_server_match and not country and not city and not hostname: - # Specific server requested (e.g., us1239) country_code = specific_server_match.group(1).upper() server_num = specific_server_match.group(2) - # Build hostname based on provider hostname = self.build_server_hostname(provider_name, country_code, server_num) - country = country_code # Set country for verification + country = country_code - # If not explicitly mapped and not a specific server, try to use query as country code elif not country and not city and not hostname: if re.match(r"^[a-z]{2}$", region): - # Convert country code to full name for Gluetun country = get_country_name(region) if not country: raise ValueError( @@ -348,13 +327,10 @@ class Gluetun(Proxy): f"Configure it in server_countries or server_cities, or use a 2-letter country code." ) - # Remove any stopped container with the same name self.remove_stopped_container(container_name) - # Find available port port = self.get_available_port() - # Create container (name already set above) try: self.create_container( container_name=container_name, @@ -366,7 +342,6 @@ class Gluetun(Proxy): hostname=hostname, ) - # Store container info self.active_containers[query_key] = { "container_name": container_name, "port": port, @@ -377,27 +352,22 @@ class Gluetun(Proxy): "hostname": hostname, } - # Wait for container to be ready (60s timeout for VPN connection) if not self.wait_for_container(container_name, timeout=60): - # Get container logs for better error message logs = self.get_container_logs(container_name, tail=30) error_msg = f"Gluetun container '{container_name}' failed to start" if hasattr(self, "_last_wait_error") and self._last_wait_error: error_msg += f": {self._last_wait_error}" if logs: - # Extract last few relevant lines log_lines = [line for line in logs.strip().split("\n") if line.strip()][-5:] error_msg += "\nRecent logs:\n" + "\n".join(log_lines) raise RuntimeError(error_msg) - # Verify IP and region if enabled if self.verify_ip: self.verify_container(query_key) return self.build_proxy_uri(port) except Exception as e: - # Cleanup on failure self.remove_container(container_name) if query_key in self.active_containers: del self.active_containers[query_key] @@ -530,15 +500,12 @@ class Gluetun(Proxy): """Check if a port is in use on the system or by any Docker container.""" import socket - # First check if the port is available on the system try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(("127.0.0.1", port)) except OSError: - # Port is in use by something on the system return True - # Also check Docker containers (in case of port forwarding) try: result = subprocess.run( ["docker", "ps", "--format", "{{.Ports}}"], @@ -566,10 +533,7 @@ class Gluetun(Proxy): Returns: Server hostname (e.g., "us1239.nordvpn.com") """ - # Convert to lowercase for hostname country_lower = country_code.lower() - - # Provider-specific hostname formats hostname_formats = { "nordvpn": f"{country_lower}{server_num}.nordvpn.com", "surfshark": f"{country_lower}-{server_num}.prod.surfshark.com", @@ -582,7 +546,6 @@ class Gluetun(Proxy): if provider_name in hostname_formats: return hostname_formats[provider_name] else: - # Generic format: country_code + server_num return f"{country_lower}{server_num}" def ensure_image_available(self, image: str = "qmcgaw/gluetun:latest") -> bool: @@ -600,7 +563,6 @@ class Gluetun(Proxy): """ log = logging.getLogger("Gluetun") - # Check if image exists locally try: result = subprocess.run( ["docker", "image", "inspect", image], @@ -619,7 +581,6 @@ class Gluetun(Proxy): log.error("Docker command not found - is Docker installed and in PATH?") return False - # Image not found, pull it log.info(f"Pulling Docker image {image}...") try: result = subprocess.run( @@ -647,7 +608,13 @@ class Gluetun(Proxy): city: Optional[str] = None, hostname: Optional[str] = None, ): - """Create and start a Gluetun Docker container.""" + """ + Create and start a Gluetun Docker container. + + Credentials go into a temporary env-file with restricted permissions and are passed with --env-file, + never as "-e KEY=VALUE" arguments, so they stay out of process listings. The file is overwritten and + deleted once "docker run" returns. + """ start_time = time.time() log_event( @@ -664,7 +631,6 @@ class Gluetun(Proxy): }, ) - # Ensure the Gluetun image is available (pulls if needed) gluetun_image = "qmcgaw/gluetun:latest" if not self.ensure_image_available(gluetun_image): log_event( @@ -679,10 +645,8 @@ class Gluetun(Proxy): credentials = provider_config.get("credentials", {}) extra_env = provider_config.get("extra_env", {}) - # Normalize provider name gluetun_provider = self.PROVIDER_MAPPING.get(provider_name.lower(), provider_name.lower()) - # Build environment variables env_vars = { "VPN_SERVICE_PROVIDER": gluetun_provider, "VPN_TYPE": vpn_type, @@ -693,20 +657,16 @@ class Gluetun(Proxy): "LOG_LEVEL": "info", } - # Add credentials if vpn_type == "wireguard": env_vars["WIREGUARD_PRIVATE_KEY"] = credentials["private_key"] - # addresses is optional - not needed for some providers like NordVPN if "addresses" in credentials: env_vars["WIREGUARD_ADDRESSES"] = credentials["addresses"] - # preshared_key is required for Windscribe, optional for others if "preshared_key" in credentials: env_vars["WIREGUARD_PRESHARED_KEY"] = credentials["preshared_key"] elif vpn_type == "openvpn": env_vars["OPENVPN_USER"] = credentials.get("username", "") env_vars["OPENVPN_PASSWORD"] = credentials.get("password", "") - # Add server location # Priority: hostname > country + city > country only # Note: Different providers support different server selection variables # - Most providers: SERVER_COUNTRIES, SERVER_CITIES @@ -733,16 +693,13 @@ class Gluetun(Proxy): if city: env_vars["SERVER_CITIES"] = city - # Add authentication if configured if self.auth_user: env_vars["HTTPPROXY_USER"] = self.auth_user if self.auth_password: env_vars["HTTPPROXY_PASSWORD"] = self.auth_password - # Merge extra environment variables env_vars.update(extra_env) - # Debug log environment variables (redact sensitive values) if debug_logger := get_debug_logger(): redact_markers = ("KEY", "PASSWORD", "PASS", "TOKEN", "SECRET", "USER") safe_env = {k: ("***" if any(m in k for m in redact_markers) else v) for k, v in env_vars.items()} @@ -753,7 +710,6 @@ class Gluetun(Proxy): context={"env_vars": safe_env, "gluetun_provider": gluetun_provider}, ) - # Build docker run command cmd = [ "docker", "run", @@ -766,12 +722,10 @@ class Gluetun(Proxy): f"127.0.0.1:{port}:8888/tcp", ] - # Avoid exposing credentials in process listings by using --env-file instead of many "-e KEY=VALUE". env_file_path: str | None = None try: fd, env_file_path = tempfile.mkstemp(prefix=f"unshackle-{container_name}-", suffix=".env") try: - # Best-effort restrictive permissions. if os.name != "nt": if hasattr(os, "fchmod"): os.fchmod(fd, 0o600) @@ -798,10 +752,8 @@ class Gluetun(Proxy): cmd.extend(["--env-file", env_file_path]) - # Add Gluetun image cmd.append(gluetun_image) - # Execute docker run try: result = subprocess.run( cmd, @@ -838,7 +790,6 @@ class Gluetun(Proxy): ) raise RuntimeError(f"Docker run failed: {error_msg}") - # Log successful container creation log_event( "gluetun_container_created", level="INFO", @@ -858,7 +809,6 @@ class Gluetun(Proxy): ) finally: if env_file_path: - # Best-effort "secure delete": overwrite then unlink (not guaranteed on all filesystems). try: with open(env_file_path, "r+b") as f: try: @@ -920,7 +870,6 @@ class Gluetun(Proxy): Dict with container info if exists and running, None otherwise """ try: - # Check if container is running if not self.is_container_running(container_name): return None @@ -945,7 +894,6 @@ class Gluetun(Proxy): port = int(port_match.group(1)) - # Extract provider and region from container name # Format: unshackle-gluetun-provider-region name_pattern = f"{self.container_prefix}-(.+)-([^-]+)$" name_match = re.match(name_pattern, container_name) @@ -955,11 +903,9 @@ class Gluetun(Proxy): provider_name = name_match.group(1) region = name_match.group(2) - # Get expected country and hostname from config (if available) country = None hostname = None - # Check if region is a specific server (e.g., us1239) specific_server_match = re.match(r"^([a-z]{2})(\d+)$", region, re.IGNORECASE) if specific_server_match: country_code = specific_server_match.group(1).upper() @@ -967,7 +913,6 @@ class Gluetun(Proxy): hostname = self.build_server_hostname(provider_name, country_code, server_num) country = country_code - # Otherwise check config elif provider_name in self.providers: provider_config = self.providers[provider_name] server_countries = provider_config.get("server_countries", {}) @@ -1014,16 +959,13 @@ class Gluetun(Proxy): while time.time() - start_time < timeout: try: - # First check if container is still running if not self.is_container_running(container_name): - # Container may have exited - check if it crashed exit_info = self.get_container_exit_info(container_name) if exit_info: last_error = f"Container exited with code {exit_info.get('exit_code', 'unknown')}" time.sleep(1) continue - # Check logs for readiness indicators result = subprocess.run( ["docker", "logs", container_name, "--tail", "100"], capture_output=True, @@ -1034,7 +976,6 @@ class Gluetun(Proxy): ) if result.returncode == 0: - # Combine stdout and stderr for checking (handle None values) stdout = result.stdout or "" stderr = result.stderr or "" all_logs = (stdout + stderr).lower() @@ -1062,7 +1003,6 @@ class Gluetun(Proxy): ) return True - # Check for fatal errors that indicate VPN connection failure error_indicators = [ "fatal", "cannot connect", @@ -1074,12 +1014,10 @@ class Gluetun(Proxy): for error in error_indicators: if error in all_logs: - # Extract the error line for better messaging for line in (stdout + stderr).split("\n"): if error in line.lower(): last_error = line.strip() break - # Fatal errors mean we should stop waiting if "fatal" in all_logs or "invalid credentials" in all_logs: return False @@ -1088,11 +1026,9 @@ class Gluetun(Proxy): time.sleep(2) - # Store the last error for potential logging if last_error: self._last_wait_error = last_error - # Log timeout/failure duration_ms = (time.time() - start_time) * 1000 log_event( "gluetun_container_wait_timeout", @@ -1157,7 +1093,8 @@ class Gluetun(Proxy): max_retries: Maximum number of retry attempts (default: 3) Raises: - RuntimeError: If verification fails after all retries + RuntimeError: If the IP resolves to a country other than the expected one, which aborts + at once with no further retries, or if verification still fails after every retry """ start_time = time.time() @@ -1182,15 +1119,12 @@ class Gluetun(Proxy): last_error = None - # Create a session with the proxy configured session = requests.Session() try: session.proxies = {"http": proxy_url, "https": proxy_url} - # Retry with exponential backoff for attempt in range(max_retries): try: - # Get external IP through the proxy using shared utility ip_info = get_ip_info(session) if ip_info: @@ -1227,7 +1161,6 @@ class Gluetun(Proxy): f"(IP: {ip_info.get('ip')}, City: {ip_info.get('city')})" ) - # Verification successful - store IP info in container record if query_key in self.active_containers: self.active_containers[query_key]["public_ip"] = ip_info.get("ip") self.active_containers[query_key]["ip_country"] = actual_country @@ -1252,11 +1185,10 @@ class Gluetun(Proxy): ) return - # ip_info was None, retry last_error = "Failed to get IP info from ipinfo.io" except RuntimeError: - raise # Re-raise region mismatch errors immediately + raise except Exception as e: last_error = str(e) log_event( @@ -1270,9 +1202,8 @@ class Gluetun(Proxy): }, ) - # Wait before retry (exponential backoff) if attempt < max_retries - 1: - wait_time = 2**attempt # 1, 2, 4 seconds + wait_time = 2**attempt time.sleep(wait_time) finally: try: @@ -1280,7 +1211,6 @@ class Gluetun(Proxy): except Exception: pass - # All retries exhausted duration_ms = (time.time() - start_time) * 1000 log_event( "gluetun_verify_failed", @@ -1331,9 +1261,7 @@ class Gluetun(Proxy): if container_name not in output: return False - # Check if container is stopped (not running) if "Exited" in output or "Created" in output or "Dead" in output: - # Container exists but is stopped - remove it subprocess.run( ["docker", "rm", "-f", container_name], capture_output=True, @@ -1353,7 +1281,6 @@ class Gluetun(Proxy): """Stop and remove a Docker container.""" try: if self.auto_cleanup: - # Use docker rm -f to force remove (stops and removes in one command) subprocess.run( ["docker", "rm", "-f", container_name], capture_output=True, @@ -1363,7 +1290,6 @@ class Gluetun(Proxy): timeout=10, ) else: - # Just stop the container subprocess.run( ["docker", "stop", container_name], capture_output=True, @@ -1373,7 +1299,6 @@ class Gluetun(Proxy): timeout=10, ) except subprocess.TimeoutExpired: - # Force kill if timeout try: subprocess.run( ["docker", "rm", "-f", container_name], diff --git a/unshackle/core/proxies/nordvpn.py b/unshackle/core/proxies/nordvpn.py index c9829c1b..986d330c 100644 --- a/unshackle/core/proxies/nordvpn.py +++ b/unshackle/core/proxies/nordvpn.py @@ -53,42 +53,37 @@ class NordVPN(Proxy): - Country ID: "228" - Specific server: "us1234" - City selection: "us:seattle", "ca:calgary" + + Returns None if the country code or ID is not one NordVPN lists. A country with no recommended + servers, or a city with no matching server, raises ValueError. """ query = query.lower() city = None - # Check if query includes city specification (e.g., "ca:calgary") if ":" in query: query, city = query.split(":", maxsplit=1) city = city.strip() if re.match(r"^[a-z]{2}\d+$", query): - # country and nordvpn server id, e.g., us1, fr1234 hostname = f"{query}.proxy.nordvpn.com" else: if query.isdigit(): - # country id country = self.get_country(by_id=int(query)) elif re.match(r"^[a-z]+$", query): - # country code country = self.get_country(by_code=query) else: raise ValueError(f"The query provided is unsupported and unrecognized: {query}") if not country: - # NordVPN doesnt have servers in this region return - # Check server_map for pinned servers (can include city) server_map_key = f"{country['code'].lower()}:{city}" if city else country["code"].lower() server_mapping = self.server_map.get(server_map_key) or ( self.server_map.get(country["code"].lower()) if not city else None ) if server_mapping: - # country was set to a specific server ID in config hostname = f"{country['code'].lower()}{server_mapping}.proxy.nordvpn.com" else: - # get the recommended server ID recommended_servers = self.get_recommended_servers(country["id"]) if not recommended_servers: raise ValueError( @@ -96,7 +91,6 @@ class NordVPN(Proxy): "Try again later. If the issue persists, double-check the query." ) - # Filter by city if specified if city: city_servers = self.filter_servers_by_city(recommended_servers, city) if not city_servers: @@ -106,7 +100,6 @@ class NordVPN(Proxy): ) recommended_servers = city_servers - # Pick a random server from the filtered list hostname = random.choice(recommended_servers)["hostname"] if hostname.startswith("gb"): @@ -148,7 +141,6 @@ class NordVPN(Proxy): filtered = [] for server in servers: - # Each server has a 'locations' list with location data locations = server.get("locations", []) for location in locations: # City data can be in different formats: @@ -156,7 +148,6 @@ class NordVPN(Proxy): # - {"city": "Seattle"} city_data = location.get("city") if city_data: - # Handle both dict and string formats city_name = city_data.get("name") if isinstance(city_data, dict) else city_data if city_name and city_name.lower() == city_lower: filtered.append(server) diff --git a/unshackle/core/proxies/surfsharkvpn.py b/unshackle/core/proxies/surfsharkvpn.py index c1988f35..9ce235ca 100644 --- a/unshackle/core/proxies/surfsharkvpn.py +++ b/unshackle/core/proxies/surfsharkvpn.py @@ -50,6 +50,9 @@ class SurfsharkVPN(Proxy): - Country ID: "228" - Specific server: "us-bos" (Boston) - City selection: "us:seattle", "ca:toronto" + + Returns None if SurfsharkVPN has no servers in the queried country. A city with no matching + server raises ValueError. """ query = query.lower() city = None @@ -64,28 +67,22 @@ class SurfsharkVPN(Proxy): hostname = f"{query}.prod.surfshark.com" else: if query.isdigit(): - # country id country = self.get_country(by_id=int(query)) elif re.match(r"^[a-z]+$", query): - # country code country = self.get_country(by_code=query) else: raise ValueError(f"The query provided is unsupported and unrecognized: {query}") if not country: - # SurfsharkVPN doesnt have servers in this region return - # Check server_map for pinned servers (can include city) server_map_key = f"{country['countryCode'].lower()}:{city}" if city else country["countryCode"].lower() server_mapping = self.server_map.get(server_map_key) or ( self.server_map.get(country["countryCode"].lower()) if not city else None ) if server_mapping: - # country was set to a specific server ID in config hostname = f"{country['code'].lower()}{server_mapping}.prod.surfshark.com" else: - # get the random server ID random_server = self.get_random_server(country["countryCode"], city) if not random_server: raise ValueError( @@ -123,7 +120,6 @@ class SurfsharkVPN(Proxy): """ servers = [x for x in self.countries if x["countryCode"].lower() == country_id.lower()] - # Filter by city if specified if city: city_lower = city.lower() # Check if servers have a 'location' field for city filtering @@ -145,7 +141,6 @@ class SurfsharkVPN(Proxy): if not servers: raise ValueError(f"Could not get random server for country '{country_id}': no servers found.") - # Only include servers that actually have a connection name to avoid KeyError. connection_names = [x["connectionName"] for x in servers if "connectionName" in x] if not connection_names: raise ValueError( diff --git a/unshackle/core/proxies/windscribevpn.py b/unshackle/core/proxies/windscribevpn.py index c136b633..78792375 100644 --- a/unshackle/core/proxies/windscribevpn.py +++ b/unshackle/core/proxies/windscribevpn.py @@ -15,6 +15,9 @@ class WindscribeVPN(Proxy): A username and password must be provided. These are Service Credentials, not your Login Credentials. The Service Credentials can be found here: https://windscribe.com/getconfig/openvpn + + server_map pins a query to a hostname. Its keys are either a country code ("ca") or a country + code with a city ("ca:toronto"), and a pinned hostname is used before any server lookup. """ if not username: raise ValueError("No Username was provided to the WindscribeVPN Proxy Service.") @@ -53,12 +56,10 @@ class WindscribeVPN(Proxy): query = query.lower() city = None - # Check if query includes city specification (e.g., "ca:toronto") if ":" in query: query, city = query.split(":", maxsplit=1) city = city.strip() - # Check server_map for pinned servers (can include city) server_map_key = f"{query}:{city}" if city else query if server_map_key in self.server_map: hostname = self.server_map[server_map_key] @@ -67,7 +68,6 @@ class WindscribeVPN(Proxy): else: server_match = re.match(r"^([a-z]{2})(\d+)$", query) if server_match: - # Specific server selection, e.g., sg007, us150 country_code, server_num = server_match.groups() hostname = self.get_specific_server(country_code, server_num) if not hostname: @@ -127,21 +127,21 @@ class WindscribeVPN(Proxy): city: Optional city name to filter by (case-insensitive) Returns: - A random hostname from matching servers, or None if none available. + A random hostname from matching servers, or None if the country has no servers. + + Raises: + ValueError: If a city was given and no server in that country matches it. """ hostnames = [] - # Collect hostnames from ALL locations matching the country code for location in self.countries: if location.get("country_code", "").lower() == country_code.lower(): for group in location.get("groups", []): - # Filter by city if specified if city: group_city = group.get("city", "") if group_city.lower() != city.lower(): continue - # Collect hostnames from this group for host in group.get("hosts", []): if hostname := host.get("hostname"): hostnames.append(hostname) @@ -149,7 +149,6 @@ class WindscribeVPN(Proxy): if hostnames: return random.choice(hostnames) elif city: - # No servers found for the specified city raise ValueError( f"No servers found in city '{city}' for country code '{country_code}'. " "Try a different city or check the city name spelling." diff --git a/unshackle/core/service.py b/unshackle/core/service.py index 8578d594..b0905d3d 100644 --- a/unshackle/core/service.py +++ b/unshackle/core/service.py @@ -138,9 +138,14 @@ def sanitize_proxy_for_log(uri: Optional[str]) -> Optional[str]: class Service(metaclass=ABCMeta): - """The Service Base Class.""" + """The Service Base Class. + + A Service must implement the abstract methods; the rest are optional overrides that fall back to the base + implementation when a Service does not define them. The main flow runs the session and authentication + methods first, then titles, then tracks and chapters. The license callbacks run later still, during track + download. + """ - # Abstract class variables ALIASES: tuple[str, ...] = () # list of aliases for the service; alternatives to the service tag. GEOFENCE: tuple[str, ...] = () # list of ip regions required to use the service. empty list == no specific region. ANIME: bool = False # service catalogue is anime; metadata lookups prefer AniList. Title.anime overrides per title. @@ -163,7 +168,6 @@ class Service(metaclass=ABCMeta): self.title_cache = TitleCacher(self.__class__.__name__) self.cache_dir = config.directories.cache / self.__class__.__name__ - # Store context for cache control flags and credential self.ctx = ctx self.credential = None # Will be set in authenticate() self.current_region = None # Will be set based on proxy/geolocation @@ -189,25 +193,21 @@ class Service(metaclass=ABCMeta): proxy_query = None proxy_provider_name = None - # Check for service-specific proxy mapping service_name = self.__class__.__name__ service_config_dict = config.services.get(service_name, {}) proxy_map = service_config_dict.get("proxy_map", {}) if proxy_map and proxy_query: - # Build the full proxy query key (e.g., "nordvpn:ca" or "us") if proxy_provider_name: full_proxy_key = f"{proxy_provider_name}:{proxy_query}" else: full_proxy_key = proxy_query - # Check if there's a mapping for this query mapped_value = proxy_map.get(full_proxy_key) if mapped_value: self.log.info( f"Found service-specific proxy mapping: {full_proxy_key} -> {sanitize_proxy_for_log(mapped_value)}" ) - # Query the proxy provider with the mapped value if proxy_provider_name: # Specific provider requested proxy_provider = next( @@ -228,7 +228,6 @@ class Service(metaclass=ABCMeta): else: self.log.warning(f"Proxy provider '{proxy_provider_name}' not found, using default proxy") else: - # No specific provider, try all providers for proxy_provider in ctx.obj.proxy_providers: mapped_proxy_uri = proxy_provider.get_proxy(mapped_value) if mapped_proxy_uri: @@ -246,7 +245,6 @@ class Service(metaclass=ABCMeta): # don't override the explicit proxy set by the user, even if they may be geoblocked with console.status("Checking if current region is Geoblocked...", spinner="dots"): if self.GEOFENCE: - # Service has geofence - need fresh IP check to determine if proxy needed try: current_region = get_ip_info(self.session)["country"].lower() if any(x.lower() == current_region for x in self.GEOFENCE): @@ -279,7 +277,6 @@ class Service(metaclass=ABCMeta): self.current_region = proxy_ip_info.get("country", "").lower() if proxy_ip_info else None except Exception as e: self.log.warning(f"Failed to verify proxy IP: {e}") - # Fallback to extracting region from proxy config self.current_region = get_region_from_proxy(proxy) else: # No proxy, use cached IP info for title caching (non-critical) @@ -365,11 +362,6 @@ class Service(metaclass=ABCMeta): # Deprecated 5.5.0 shim for service repos still on the old underscored name; drop once they migrate. _get_tracks_for_variants = get_tracks_for_variants - # Optional Abstract functions - # The following functions may be implemented by the Service. - # Otherwise, the base service code (if any) of the function will be executed on call. - # The functions will be executed in shown order. - @staticmethod def get_session() -> requests.Session: """ @@ -527,10 +519,6 @@ class Service(metaclass=ABCMeta): """ return None - # Required Abstract functions - # The following functions *must* be implemented by the Service. - # The functions will be executed in shown order. - @abstractmethod def get_titles(self) -> Titles_T: """ @@ -560,7 +548,6 @@ class Service(metaclass=ABCMeta): Returns: Titles object (Movies, Series, or Album) """ - # Try to get title_id from service instance if not provided if title_id is None: # Different services store the title ID in different attributes if hasattr(self, "title"): @@ -572,7 +559,6 @@ class Service(metaclass=ABCMeta): self.log.debug("Cannot determine title_id for caching, bypassing cache") return self.apply_title_map(self.get_titles()) - # Get cache control flags from context no_cache = False reset_cache = False if self.ctx and self.ctx.parent: @@ -582,7 +568,6 @@ class Service(metaclass=ABCMeta): # Get account hash for cache key account_hash = get_account_hash(self.credential) - # Use title cache to get titles with fallback support titles = self.title_cache.get_cached_titles( title_id=str(title_id), fetch_function=self.get_titles, diff --git a/unshackle/core/service_repo.py b/unshackle/core/service_repo.py index 1d708b05..6ec919ef 100644 --- a/unshackle/core/service_repo.py +++ b/unshackle/core/service_repo.py @@ -30,7 +30,6 @@ def stamp_for(dest: Path) -> Path: return dest.parent / f"{dest.name}{STAMP_SUFFIX}" -# owner/repo shorthand: exactly two path-like segments, no scheme SHORTHAND_RE = re.compile(r"^[\w.-]+/[\w.-]+$") @@ -147,10 +146,10 @@ def refresh_repo(spec: str) -> tuple[Optional[Path], list[str]]: if not binaries.Git: return dest, [] old = head(dest) - discarded = local_dirty_services(dest) # uncommitted edits the force reset is about to wipe + discarded = local_dirty_services(dest) force_sync(dest, stamp) lines = [f"! {tag} (local changes discarded)" for tag in discarded] - lines += changed_services(dest, old, head(dest)) # upstream changes pulled in + lines += changed_services(dest, old, head(dest)) return dest, lines diff --git a/unshackle/core/services.py b/unshackle/core/services.py index 97195aa3..c318322b 100644 --- a/unshackle/core/services.py +++ b/unshackle/core/services.py @@ -35,7 +35,6 @@ for _entry in raw: else: service_dirs.append(Path(_entry)) -# dedupe by tag honoring list order (first wins); track shadows for the load summary seen: dict[str, Path] = {} SHADOWED: list[str] = [] for _dir in service_dirs: @@ -127,8 +126,6 @@ class Services(click.Group): remote_services_cache: list[dict] | None = None - # Click-specific methods - def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: """Preprocess --slow to support optional range value before Click parses args.""" processed = [] @@ -300,8 +297,6 @@ class Services(click.Group): log.debug(f"Remote service info lookup failed for {tag}: {e!r}") return None - # Methods intended to be used anywhere - @staticmethod def get_tags() -> list[str]: """Returns a list of service tags from all available Services.""" diff --git a/unshackle/core/session.py b/unshackle/core/session.py index 5e42eb7c..8faddb8d 100644 --- a/unshackle/core/session.py +++ b/unshackle/core/session.py @@ -20,9 +20,6 @@ from requests.structures import CaseInsensitiveDict from unshackle.core.config import config -# --------------------------------------------------------------------------- -# Shared retry / pool / timeout policy: single source of truth -# --------------------------------------------------------------------------- # rnet's only distinguishing role is TLS fingerprinting; retry, backoff, # pooling, and timeouts must match the plain-requests path. Both RnetSession # (below) and Service.get_session import these so the two can't drift. @@ -39,10 +36,6 @@ CONNECT_TIMEOUT = 10 READ_TIMEOUT = 30 -# --------------------------------------------------------------------------- -# Impersonate preset mapping — rnet uses named presets (no custom JA3/Akamai) -# --------------------------------------------------------------------------- - DEFAULT_IMPERSONATE = rnet.Impersonate.Chrome131 @@ -62,7 +55,6 @@ def resolve_impersonate(browser: str) -> rnet.Impersonate: ) -# Map string method names to rnet.Method enum METHOD_MAP: dict[str, rnet.Method] = { "GET": rnet.Method.GET, "POST": rnet.Method.POST, @@ -75,11 +67,6 @@ METHOD_MAP: dict[str, rnet.Method] = { } -# --------------------------------------------------------------------------- -# Response headers adapter — bytes → str -# --------------------------------------------------------------------------- - - class RnetResponseHeaders(MutableMapping): """Read-only str-based view over rnet's bytes-based HeaderMap.""" @@ -125,11 +112,6 @@ class RnetResponseHeaders(MutableMapping): return [(self.decode(k), self.decode(v)) for k, v in self._map.items()] -# --------------------------------------------------------------------------- -# Response wrapper — requests-compatible interface -# --------------------------------------------------------------------------- - - class RnetResponse: """Wraps rnet.BlockingResponse with a requests-compatible API.""" @@ -245,11 +227,6 @@ class RnetResponse: pass -# --------------------------------------------------------------------------- -# Session headers adapter — persists via client.update() -# --------------------------------------------------------------------------- - - class RnetSessionHeaders(CaseInsensitiveDict): """Dict-like headers that persist to the rnet client via update().""" @@ -288,11 +265,6 @@ class RnetSessionHeaders(CaseInsensitiveDict): super().__delitem__(key) -# --------------------------------------------------------------------------- -# Session cookies adapter -# --------------------------------------------------------------------------- - - class RnetCookieAdapter(MutableMapping): """Cookie adapter that bridges requests-style cookie access to rnet.""" @@ -455,11 +427,6 @@ class RnetCookieAdapter(MutableMapping): return list(self._flat.values()) -# --------------------------------------------------------------------------- -# Session proxy adapter -# --------------------------------------------------------------------------- - - class RnetProxyDict(dict): """Dict-like proxy config that syncs to the rnet client. @@ -488,22 +455,12 @@ class RnetProxyDict(dict): self.sync() -# --------------------------------------------------------------------------- -# Exceptions -# --------------------------------------------------------------------------- - - class MaxRetriesError(Exception): def __init__(self, message: str, cause: Optional[Exception] = None) -> None: super().__init__(message) self.__cause__ = cause -# --------------------------------------------------------------------------- -# RnetSession — main session class -# --------------------------------------------------------------------------- - - class RnetSession: """TLS-fingerprinted HTTP session powered by rnet (Rust/BoringSSL). @@ -658,29 +615,30 @@ class RnetSession: return min(sleep_time, self.max_backoff) def request(self, method: str, url: str, **kwargs: Any) -> RnetResponse: + """Send a request, retrying on the status forcelist and on the caught exception types. + + Only methods in allowed_methods are retried; any other method gets a single attempt. + A max_retries kwarg overrides the session retry budget for this call alone, where 0 means + one attempt with no retries. Once the budget is spent, MaxRetriesError is raised with the + last failure as its cause. + """ client = self.ensure_client() method_upper = method.upper() if isinstance(method, str) else str(method).upper() - # Per-request override of the session retry budget (rnet.request() would reject the kwarg). - # max_retries=0 means a single attempt with no session-level retries. max_retries = kwargs.pop("max_retries", None) if max_retries is None: max_retries = self.max_retries - # Build URL with params url = self.build_url(url, kwargs.pop("params", None)) - # Default allow_redirects=True kwargs.setdefault("allow_redirects", True) - # Pass verify setting if not self.verify: kwargs.setdefault("verify", False) # Remove kwargs rnet doesn't understand kwargs.pop("stream", None) # rnet responses are always lazy - # Translate requests-compatible 'data' kwarg to rnet equivalents data = kwargs.pop("data", None) if data is not None: if isinstance(data, dict): @@ -690,7 +648,6 @@ class RnetSession: else: kwargs["body"] = data - # Resolve method enum rnet_method = METHOD_MAP.get(method_upper) if rnet_method is None: raise ValueError(f"Unsupported HTTP method: {method}") @@ -699,7 +656,6 @@ class RnetSession: if kwargs.get("headers") is not None: kwargs["headers"] = dict(kwargs["headers"]) - # Skip retry for non-allowed methods if method_upper not in self.allowed_methods: raw_resp = client.request(rnet_method, url, **kwargs) return RnetResponse(raw_resp) @@ -712,7 +668,6 @@ class RnetSession: raw_resp = client.request(rnet_method, url, **kwargs) response = RnetResponse(raw_resp) - # Log requests when debug_requests is enabled if config.debug_requests: parsed_url = urlparse(url) port_str = f":{parsed_url.port}" if parsed_url.port else "" @@ -808,11 +763,6 @@ class RnetSession: pass -# --------------------------------------------------------------------------- -# session() factory -# --------------------------------------------------------------------------- - - def session( browser: Optional[str] = None, **kwargs: Any, diff --git a/unshackle/core/themes.py b/unshackle/core/themes.py index 56f760f8..dae47478 100644 --- a/unshackle/core/themes.py +++ b/unshackle/core/themes.py @@ -18,7 +18,7 @@ PALETTES: dict[str, dict[str, str]] = { "catppuccin-mocha": { "bg": "rgb(30,30,46)", "text": "rgb(205,214,244)", - "text2": "rgb(162,169,193)", # slightly darker + "text2": "rgb(162,169,193)", "black": "rgb(69,71,90)", "bright_black": "rgb(88,91,112)", "red": "rgb(243,139,168)", diff --git a/unshackle/core/title_cacher.py b/unshackle/core/title_cacher.py index 3726518b..33a9c155 100644 --- a/unshackle/core/title_cacher.py +++ b/unshackle/core/title_cacher.py @@ -46,18 +46,14 @@ class TitleCacher: # This ensures consistent length and filesystem-safe keys title_hash = hashlib.sha256(title_id.encode()).hexdigest()[:16] - # Start with base key using hash key_parts = ["titles", title_hash] - # Add region if available if region: key_parts.append(region.lower()) - # Add account hash if available if account_hash: key_parts.append(account_hash[:8]) # Use first 8 chars of hash - # Join with underscores cache_key = "_".join(key_parts) return cache_key @@ -85,41 +81,33 @@ class TitleCacher: Returns: Titles object (Movies, Series, or Album) """ - # If caching is globally disabled or no_cache flag is set if not config.title_cache_enabled or no_cache: self.no_cache = True self.log.debug("Cache bypassed, fetching fresh titles") return fetch_function() - # Generate cache key cache_key = self.generate_cache_key(title_id, region, account_hash) - # If reset_cache flag is set, clear the cache entry if reset_cache: self.log.info(f"Clearing cache for {cache_key}") cache_path = (config.directories.cache / self.service_name / cache_key).with_suffix(".json") if cache_path.exists(): cache_path.unlink() - # Try to get from cache cache = self.cacher.get(cache_key, version=1) - # Check if we have valid cached data if cache and not cache.expired: self.stats["hits"] += 1 self.log.debug(f"Cache hit for {title_id} (hits: {self.stats['hits']}, misses: {self.stats['misses']})") return cache.data - # Cache miss or expired, try to fetch fresh data self.stats["misses"] += 1 self.log.debug(f"Cache miss for {title_id} fetching fresh data") try: - # Attempt to fetch fresh titles titles = fetch_function() if titles: - # Successfully fetched, update cache self.log.debug(f"Successfully fetched titles for {title_id}, updating cache") cache = self.cacher.get(cache_key, version=1) cache.set(titles, expiration=datetime.now() + timedelta(seconds=config.title_cache_time)) @@ -127,9 +115,7 @@ class TitleCacher: return titles except Exception as e: - # API call failed, check if we have fallback cached data if cache and cache.data: - # We have expired cached data, use it as fallback current_time = datetime.now() max_retention_time = cache.expiration + timedelta( seconds=config.title_cache_max_retention - config.title_cache_time @@ -143,7 +129,6 @@ class TitleCacher: ) self.log.debug(f"Error was: {e}") - # Extend cache lifetime extended_expiration = current_time + timedelta(minutes=5) if extended_expiration < max_retention_time: cache.expiration = extended_expiration @@ -153,7 +138,6 @@ class TitleCacher: else: self.log.error(f"API call failed and cached data for {title_id} exceeded maximum retention time") - # Re-raise the exception if no fallback available raise def clear_all_title_cache(self): @@ -179,8 +163,6 @@ class TitleCacher: "hit_rate": f"{hit_rate:.1f}%", } - # -- Generic provider cache methods -- - def get_cached_provider( self, provider_name: str, @@ -217,7 +199,6 @@ class TitleCacher: self.log.debug(f"{provider_name} cache hit for {title_id}") - # Return the inner data (provider-specific format) response = provider_data.get("response") if response is not None: return response @@ -272,7 +253,6 @@ class TitleCacher: "expires_at": now + timedelta(days=ttl_days), } else: - # Generic format: store data directly with metadata cache_entry = { "response": data, "kind": kind, @@ -298,21 +278,16 @@ def get_region_from_proxy(proxy_url: Optional[str]) -> Optional[str]: if not proxy_url: return None - # Try to extract region from common proxy patterns - # e.g., "us123.nordvpn.com", "gb-proxy.example.com" import re - # Pattern for NordVPN style nord_match = re.search(r"([a-z]{2})\d+\.nordvpn", proxy_url.lower()) if nord_match: return nord_match.group(1) - # Pattern for country code at start cc_match = re.search(r"([a-z]{2})[-_]", proxy_url.lower()) if cc_match: return cc_match.group(1) - # Pattern for country code subdomain subdomain_match = re.search(r"://([a-z]{2})\.", proxy_url.lower()) if subdomain_match: return subdomain_match.group(1) @@ -333,11 +308,9 @@ def get_account_hash(credential) -> Optional[str]: if not credential: return None - # Use existing sha1 property if available if hasattr(credential, "sha1"): return credential.sha1 - # Otherwise generate hash from username if hasattr(credential, "username"): return hashlib.sha1(credential.username.encode()).hexdigest() diff --git a/unshackle/core/titles/episode.py b/unshackle/core/titles/episode.py index d8b6daae..860a9be5 100644 --- a/unshackle/core/titles/episode.py +++ b/unshackle/core/titles/episode.py @@ -90,7 +90,6 @@ class Episode(Title): if name is not None: name = name.strip() - # ignore episode names that are the episode number or title name if re.match(r"Episode ?#?\d+", name, re.IGNORECASE): name = None elif name.lower() == title.lower(): diff --git a/unshackle/core/titles/music.py b/unshackle/core/titles/music.py index a6b0d1b1..1fe20526 100644 --- a/unshackle/core/titles/music.py +++ b/unshackle/core/titles/music.py @@ -166,12 +166,10 @@ class Song(Title): def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str: if folder: - # Album folder name: prefer the dedicated "albums" template, fall back to the - # legacy "songs" folder template, then to "{artist} - {album} ({year})". template = config.get_folder_template("albums") or config.get_folder_template("songs") if template: context = self.build_template_context(media_info, show_service) - spacer = detect_spacer(template) # one style for the whole path + spacer = detect_spacer(template) segments = [ TemplateFormatter(seg, spacer).format(context) for seg in re.split(r"[\\/]", template) diff --git a/unshackle/core/tracks/attachment.py b/unshackle/core/tracks/attachment.py index b970768c..3ea7839c 100644 --- a/unshackle/core/tracks/attachment.py +++ b/unshackle/core/tracks/attachment.py @@ -59,7 +59,6 @@ class Attachment: parsed_url = urlparse(url) file_name = os.path.basename(parsed_url.path) or "attachment" - # Use provided name for the file if available if name: safe_name = re.sub(r'[<>:"/\\|?*]', "", name).replace(" ", "_") file_name = f"{safe_name}{os.path.splitext(file_name)[1]}" diff --git a/unshackle/core/tracks/hybrid.py b/unshackle/core/tracks/hybrid.py index 538ade44..4197f51a 100644 --- a/unshackle/core/tracks/hybrid.py +++ b/unshackle/core/tracks/hybrid.py @@ -39,7 +39,6 @@ class Hybrid: self.hdr10plus_to_dv = False self.hdr10plus_file = "HDR10Plus.json" - # Get resolution info from HDR10 track for display hdr10_track = next((v for v in videos if v.range == Video.Range.HDR10), None) hdr10p_track = next((v for v in videos if v.range == Video.Range.HDR10P), None) track_for_res = hdr10_track or hdr10p_track @@ -63,7 +62,6 @@ class Hybrid: if not video.path or not os.path.exists(video.path): raise ValueError(f"Video track {video.id} was not downloaded before injection.") - # Check if we have DV track available has_dv = any(video.range == Video.Range.DV for video in self.videos) has_hdr10 = any(video.range == Video.Range.HDR10 for video in self.videos) has_hdr10p = any(video.range == Video.Range.HDR10P for video in self.videos) @@ -71,7 +69,6 @@ class Hybrid: if not has_hdr10 and not has_hdr10p: raise ValueError("No HDR10 or HDR10+ track available for hybrid processing.") - # If we have HDR10+ but no DV, we can convert HDR10+ to DV if not has_dv and has_hdr10p: console.status("No DV track found, but HDR10+ is available. Will convert HDR10+ to DV.") self.hdr10plus_to_dv = True @@ -83,7 +80,6 @@ class Hybrid: return for video in videos: - # Use the actual path from the video track save_path = video.path if not save_path or not os.path.exists(save_path): raise ValueError(f"Video track {video.id} was not downloaded or path not found: {save_path}") @@ -97,12 +93,10 @@ class Hybrid: self.extract_stream(save_path, "DV") if self.hdr10plus_to_dv: - # Extract HDR10+ metadata and convert to DV hdr10p_video = next(v for v in videos if v.range == Video.Range.HDR10P) self.extract_hdr10plus(hdr10p_video) self.convert_hdr10plus_to_dv() else: - # Regular DV extraction dv_video = next(v for v in videos if v.range == Video.Range.DV) self.extract_rpu(dv_video) if os.path.isfile(config.directories.temp / "RPU_UNT.bin"): @@ -112,7 +106,6 @@ class Hybrid: # RPU already extracted with mode 3 pass - # Edit L6 with actual luminance values from RPU, then L5 active area self.level_6() base_video = next((v for v in videos if v.range in (Video.Range.HDR10, Video.Range.HDR10P)), None) if base_video and base_video.path: @@ -218,7 +211,6 @@ class Hybrid: ffprobe_bin = str(FFProbe) if FFProbe else "ffprobe" ffmpeg_bin = str(FFMPEG) if FFMPEG else "ffmpeg" - # Get video duration for random sampling with console.status("Detecting active area (crop detection)...", spinner="dots"): result_duration = subprocess.run( [ffprobe_bin, "-v", "error", "-show_entries", "format=duration", "-of", "json", str(input_video)], @@ -242,7 +234,6 @@ class Hybrid: duration_info = json.loads(result_duration.stdout) duration = float(duration_info["format"]["duration"]) - # Get video resolution for proper border calculation result_streams = subprocess.run( [ ffprobe_bin, @@ -277,7 +268,6 @@ class Hybrid: original_width = int(stream_info["streams"][0]["width"]) original_height = int(stream_info["streams"][0]["height"]) - # Sample 10 random timestamps and run cropdetect on each random_times = sorted(random.uniform(0, duration) for _ in range(10)) crop_results = [] @@ -316,7 +306,6 @@ class Hybrid: if crop_match: w, h = int(crop_match.group(1)), int(crop_match.group(2)) x, y = int(crop_match.group(3)), int(crop_match.group(4)) - # Calculate actual border sizes from crop geometry left = x top = y right = original_width - w - x @@ -333,12 +322,11 @@ class Hybrid: self.log.warning("No crop data detected, skipping L5") return - # Find the most common crop values crop_counts = {} for crop in crop_results: crop_counts[crop] = crop_counts.get(crop, 0) + 1 most_common = max(crop_counts, key=crop_counts.get) - left, top, right, bottom = most_common # frame instead of leaving phantom bars from the source. + left, top, right, bottom = most_common l5_json = { "active_area": { @@ -651,11 +639,9 @@ class Hybrid: return with console.status("Converting HDR10+ metadata to Dolby Vision...", spinner="dots"): - # Extract actual HDR metadata from the source stream max_mdl, min_mdl, max_cll, max_fall = self.probe_hdr_metadata() max_mdl, min_mdl, max_cll, max_fall = self.sanitize_l6(max_mdl, min_mdl, max_cll, max_fall) - # First create the extra metadata JSON for dovi_tool extra_metadata = { "cm_version": "V29", "length": 0, # dovi_tool will figure this out @@ -693,6 +679,5 @@ class Hybrid: success=True, ) - # Clean up temporary files Path.unlink(config.directories.temp / "extra.json") Path.unlink(config.directories.temp / self.hdr10plus_file) diff --git a/unshackle/core/tracks/subtitle.py b/unshackle/core/tracks/subtitle.py index 75fa87d4..d9e619c5 100644 --- a/unshackle/core/tracks/subtitle.py +++ b/unshackle/core/tracks/subtitle.py @@ -284,11 +284,8 @@ class Subtitle(Track): timescale=1, # ? ) - # Sanitize WebVTT timestamps before parsing text = Subtitle.sanitize_webvtt_timestamps(text) - # Remove cue identifiers that confuse parsers like pysubs2 text = Subtitle.sanitize_webvtt_cue_identifiers(text) - # Merge overlapping cues with line positioning into single multi-line cues text = Subtitle.merge_overlapping_webvtt_cues(text) preserve_formatting = config.subtitle.get("preserve_formatting", True) @@ -306,7 +303,6 @@ class Subtitle(Track): # some renditions carry headers but no cues; write them out rather than fail the download self.path.write_text(text, encoding="utf8") except pycaption.exceptions.CaptionReadSyntaxError: - # If first attempt fails, try more aggressive sanitization text = Subtitle.sanitize_webvtt(text) try: caption_set = pycaption.WebVTTReader().read(text) @@ -340,7 +336,6 @@ class Subtitle(Track): Returns: Sanitized WebVTT content """ - # Replace negative timestamps with 00:00:00.000 return re.sub(r"(-\d+:\d+:\d+\.\d+)", "00:00:00.000", text) @staticmethod @@ -359,7 +354,6 @@ class Subtitle(Track): for i, line in enumerate(lines): line = line.strip() if Subtitle.CUE_ID_PATTERN.match(line): - # Look ahead to see if next non-empty line is a timing line j = i + 1 while j < len(lines) and not lines[j].strip(): j += 1 @@ -393,14 +387,11 @@ class Subtitle(Track): while i < len(lines): line = lines[i].strip() - # Check if this line is a cue identifier followed by a timing line if Subtitle.CUE_ID_PATTERN.match(line): - # Look ahead to see if next non-empty line is a timing line j = i + 1 while j < len(lines) and not lines[j].strip(): j += 1 if j < len(lines) and ("-->" in lines[j] or Subtitle.TIMING_START_PATTERN.match(lines[j].strip())): - # This is a cue identifier, skip it i += 1 continue @@ -451,7 +442,6 @@ class Subtitle(Track): start_str, end_str = match.group(1), match.group(2) timings.append((Subtitle.parse_vtt_time(start_str), Subtitle.parse_vtt_time(end_str))) - # Check for overlapping cues (within 50ms start, same end) for i in range(len(timings) - 1): curr_start, curr_end = timings[i] next_start, next_end = timings[i + 1] @@ -542,9 +532,7 @@ class Subtitle(Track): break if len(group) > 1: - # Sort by line position (lower % = higher on screen = first) group.sort(key=lambda x: x["line_pos"]) - # Use the earliest start time from the group earliest = min(group, key=lambda x: x["start_ms"]) merged_cues.append( { @@ -582,6 +570,10 @@ class Subtitle(Track): """ More thorough sanitization of WebVTT files to handle multiple potential issues. + This is lossy, so use it only as a fallback once normal parsing has failed. Everything before the + WEBVTT header line is discarded and the header itself is reduced to a bare "WEBVTT" line. Negative + timestamps become 00:00:00.000 and timestamps missing the hours field are padded to HH:MM:SS.mmm. + Parameters: text: The WebVTT content as string @@ -596,7 +588,6 @@ class Subtitle(Track): sanitized_lines = [] timestamp_pattern = re.compile(r"^((?:\d+:)?\d+:\d+\.\d+)\s+-->\s+((?:\d+:)?\d+:\d+\.\d+)") - # Skip invalid headers - keep only WEBVTT header_done = False for line in lines: if not header_done: @@ -605,17 +596,14 @@ class Subtitle(Track): header_done = True continue - # Replace negative timestamps if "-" in line and "-->" in line: line = re.sub(r"(-\d+:\d+:\d+\.\d+)", "00:00:00.000", line) - # Validate timestamp format match = timestamp_pattern.match(line) if match: start_time = match.group(1) end_time = match.group(2) - # Ensure proper format with hours if missing if start_time.count(":") == 1: start_time = f"00:{start_time}" if end_time.count(":") == 1: @@ -768,28 +756,22 @@ class Subtitle(Track): i = 0 while i < len(lines): - # Skip empty lines if not lines[i].strip(): sanitized_lines.append(lines[i]) i += 1 continue - # Check for timestamp lines if "-->" in lines[i]: - # Validate timestamp format timestamp_parts = lines[i].split("-->") if len(timestamp_parts) != 2 or not timestamp_parts[1].strip() or timestamp_parts[1].strip() == "0": - # Skip this timestamp and its content until next timestamp or end j = i + 1 while j < len(lines) and "-->" not in lines[j] and lines[j].strip(): j += 1 i = j continue - # Add valid timestamp line sanitized_lines.append(lines[i]) else: - # Add non-timestamp line sanitized_lines.append(lines[i]) i += 1 @@ -909,14 +891,10 @@ class Subtitle(Track): # media if box.type == b"styp": - # essentially the start of each segment - # media var resets - # > tfhd + # styp opens a new segment, so reset per-segment fragment state default_duration = None - # > tfdt saw_tfdt_box = False base_time = 0 - # > trun saw_trun_box = False samples = [] @@ -963,7 +941,6 @@ class Subtitle(Track): for cue_box in vttc_box.children: if cue_box.type == b"vsid": - # this is a V(?) Source ID box, we don't care continue if cue_box.type == b"sttg": layout = Layout(webvtt_positioning=cue_box.settings) @@ -1009,11 +986,9 @@ class Subtitle(Track): if not self.path or not self.path.exists(): raise ValueError("You must download the subtitle track first.") - # Check configuration for SDH stripping method sdh_method = config.subtitle.get("sdh_method", "auto") if sdh_method == "subby" and self.codec == Subtitle.Codec.SubRip: - # Use subby's SDHStripper directly on the file fixer = CommonIssuesFixer() stripper = SDHStripper() srt, _ = fixer.from_file(self.path) @@ -1022,10 +997,8 @@ class Subtitle(Track): stripped.save(self.path) return elif sdh_method == "subtitleedit" and binaries.SubtitleEdit: - # Force use of SubtitleEdit pass # Continue to SubtitleEdit section below elif sdh_method == "filter-subs": - # Force use of filter-subs sub = Subtitles(self.path) try: sub.filter(rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=True, rm_author=True) @@ -1042,7 +1015,6 @@ class Subtitle(Track): sub.save() return elif sdh_method == "auto": - # Try subby first for SRT files, then fall back if self.codec == Subtitle.Codec.SubRip: try: fixer = CommonIssuesFixer() diff --git a/unshackle/core/tracks/subtitle_convert.py b/unshackle/core/tracks/subtitle_convert.py index 964ee5d7..fca380d2 100644 --- a/unshackle/core/tracks/subtitle_convert.py +++ b/unshackle/core/tracks/subtitle_convert.py @@ -51,7 +51,6 @@ PYCAPTION_WRITERS = { Codec.WebVTT: pycaption.WebVTTWriter, } -# pysubs2 format identifiers per codec. PYSUBS2_FORMATS: dict[Codec, str] = { Codec.SubRip: "srt", Codec.SubStationAlpha: "ssa", diff --git a/unshackle/core/tracks/track.py b/unshackle/core/tracks/track.py index 88559a3b..48ccd857 100644 --- a/unshackle/core/tracks/track.py +++ b/unshackle/core/tracks/track.py @@ -60,7 +60,11 @@ def direct_session(session: Union[Session, "RnetSession"]) -> Session: def read_top_level_box(path: Path, box_type: bytes) -> Optional[bytes]: - # seek through top-level box headers; only the wanted box's bytes are read into RAM + """Read a single top-level box from an MP4 file. + + Only box headers are walked, so the wanted box is the only payload read into memory. Returns None if the + file has no such top-level box, or if a box header is truncated or declares an impossible size. + """ file_size = path.stat().st_size with path.open("rb") as f: start = 0 @@ -69,12 +73,12 @@ def read_top_level_box(path: Path, box_type: bytes) -> Optional[bytes]: if len(header) < 8: return None size = int.from_bytes(header[:4], "big") - if size == 1: # 64-bit largesize + if size == 1: large = f.read(8) if len(large) < 8: return None size = int.from_bytes(large, "big") - elif size == 0: # box runs to EOF + elif size == 0: size = file_size - start if size < 8: return None @@ -177,15 +181,14 @@ def assert_fragments_decrypted(path: Path) -> None: break size = int.from_bytes(header[:4], "big") box_header = 8 - if size == 1: # 64-bit largesize + if size == 1: large = f.read(8) if len(large) < 8: break size = int.from_bytes(large, "big") box_header = 16 - elif size == 0: # box runs to EOF + elif size == 0: size = file_size - start - # below the consumed header the cursor stalls and desyncs onto payload if size < box_header or start + size > file_size: logging.getLogger("track").warning( f"{path.name}: malformed box size at offset {start}; cannot verify the " @@ -247,7 +250,7 @@ class DownloadContext: class Track: class Descriptor(Enum): - URL = 1 # Direct URL, nothing fancy + URL = 1 HLS = 2 # https://en.wikipedia.org/wiki/HTTP_Live_Streaming DASH = 3 # https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP ISM = 4 # https://learn.microsoft.com/en-us/silverlight/smooth-streaming @@ -328,7 +331,7 @@ class Track: if self.name is None: lang = Language.get(self.language) if (lang.language or "").lower() == (lang.territory or "").lower(): - lang.territory = None # e.g. en-en, de-DE + lang.territory = None reduced = lang.simplify_script() extra_parts = [] if reduced.script is not None: @@ -409,7 +412,11 @@ class Track: adaptive_workers: bool = False, download_processes: int = 1, ): - """Download and optionally Decrypt this Track.""" + """Download and optionally Decrypt this Track. + + A URL-descriptor Video or Audio track with no `drm` set has its DRM probed from the init data of the + stream and stored on the track, so a service need not declare `drm` itself. + """ from unshackle.core.manifests import DASH, HLS, ISM if DOWNLOAD_LICENCE_ONLY.is_set(): @@ -485,8 +492,6 @@ class Track: elif self.descriptor == self.Descriptor.URL: try: if not self.drm and track_type in ("Video", "Audio"): - # the service might not have explicitly defined the `drm` property - # try find DRM information from the init data of URL based on CDM type if is_playready_cdm(cdm): try: self.drm = [PlayReady.from_track(self, session)] @@ -512,21 +517,18 @@ class Track: track_kid = self.get_key_id(session=session) drm = self.get_drm_for_cdm(cdm) if isinstance(drm, Widevine): - # license and grab content keys if not prepare_drm: raise ValueError("prepare_drm func must be supplied to use Widevine DRM") progress(downloaded="LICENSING") prepare_drm(drm, track_kid=track_kid) progress(downloaded="[yellow]LICENSED") elif isinstance(drm, PlayReady): - # license and grab content keys if not prepare_drm: raise ValueError("prepare_drm func must be supplied to use PlayReady DRM") progress(downloaded="LICENSING") prepare_drm(drm, track_kid=track_kid) progress(downloaded="[yellow]LICENSED") elif isinstance(drm, ClearKeyCENC): - # license and grab content keys (no CDM involved) if not prepare_drm: raise ValueError("prepare_drm func must be supplied to use ClearKey DRM") progress(downloaded="LICENSING") @@ -598,7 +600,6 @@ class Track: raise if DOWNLOAD_CANCELLED.is_set(): - # we stopped during the download, let's exit return if not DOWNLOAD_LICENCE_ONLY.is_set(): @@ -750,7 +751,7 @@ class Track: return tenc.key_ID for uuid_box in get_boxes(init_data, b"uuid"): - if uuid_box.extended_type == UUID("8974dbce-7be7-4c51-84f9-7148f9882554"): # tenc + if uuid_box.extended_type == UUID("8974dbce-7be7-4c51-84f9-7148f9882554"): tenc = uuid_box.data if tenc.key_ID.int != 0: return tenc.key_ID @@ -962,11 +963,10 @@ class Track: args.extend( [ - # Following are very important! "-map_metadata", - "-1", # don't transfer metadata to output file + "-1", "-fflags", - "bitexact", # only have minimal tag data, reproducible mux + "bitexact", "-codec", "copy", ] @@ -995,7 +995,6 @@ class Track: bsf_applied = bsf_v is not None except subprocess.CalledProcessError as e: if b"Malformed AAC bitstream detected" in e.stderr: - # e.g., TruTV's dodgy encodes ffmpeg(["-y", "-bsf:a", "aac_adtstoasc"], bsf=bsf_v) bsf_applied = bsf_v is not None elif bsf_v is not None: diff --git a/unshackle/core/tracks/tracks.py b/unshackle/core/tracks/tracks.py index 2a09761f..8c5c6aab 100644 --- a/unshackle/core/tracks/tracks.py +++ b/unshackle/core/tracks/tracks.py @@ -191,7 +191,7 @@ class Tracks: def exists(self, by_id: Optional[str] = None, by_url: Optional[Union[str, list[str]]] = None) -> bool: """Check if a track already exists by various methods.""" - if by_id: # recommended + if by_id: return any(x.id == by_id for x in self) if by_url: return any(x.url == by_url for x in self) @@ -251,7 +251,6 @@ class Tracks: return # resolution first, then bitrate (unknown-bitrate tracks still rank by resolution) self.videos.sort(key=lambda x: (x.height or 0, float(x.bitrate or 0.0)), reverse=True) - # language for language in reversed(by_language or []): if str(language) in ("all", "best"): language = next((x.language for x in self.videos if x.is_original_lang), "") @@ -279,9 +278,7 @@ class Tracks: self.audio.sort(key=lambda x: rank.get(x.codec.name if x.codec else "", default_rank)) # Atmos tracks first (prioritize over higher bitrate non-Atmos) self.audio.sort(key=lambda x: not x.atmos) - # descriptive tracks last self.audio.sort(key=lambda x: x.descriptive) - # language for language in reversed(by_language or []): if str(language) in ("all", "best"): language = next((x.language for x in self.audio if x.is_original_lang), "") @@ -349,7 +346,6 @@ class Tracks: else: by_lang() by_type() - # sections for language in reversed(by_language or []): if str(language) in ("all", "best"): language = next((x.language for x in self.subtitles if x.is_original_lang), "") @@ -495,6 +491,9 @@ class Tracks: """ Multiplex all the Tracks into a Matroska Container file. + A failed mux does not raise. mkvmerge's exit code and error lines come back alongside the + output path, and the caller must check them. + Parameters: title: Set the Matroska Container file title. Usually displayed in players instead of the filename if set. @@ -580,7 +579,6 @@ class Tracks: else: is_default = i == 0 - # Prepare base arguments video_args = [ "--language", f"0:{vt.language}", @@ -589,7 +587,7 @@ class Tracks: "--original-flag", f"0:{vt.is_original_lang}", "--compression", - "0:none", # disable extra compression + "0:none", ] # Add FPS fix if needed (typically for hybrid mode to prevent sync issues) @@ -646,7 +644,7 @@ class Tracks: "--original-flag", f"0:{at.is_original_lang}", "--compression", - "0:none", # disable extra compression + "0:none", "(", str(at.path), ")", @@ -679,7 +677,7 @@ class Tracks: "--original-flag", f"0:{st.is_original_lang}", "--compression", - "0:none", # disable extra compression (probably zlib) + "0:none", "(", str(st.path), ")", @@ -755,7 +753,6 @@ class Tracks: }, ) - # let potential failures go to caller, caller should handle try: errors = [] warnings = [] diff --git a/unshackle/core/utilities.py b/unshackle/core/utilities.py index 0870b8a7..38dffeab 100644 --- a/unshackle/core/utilities.py +++ b/unshackle/core/utilities.py @@ -80,7 +80,6 @@ def rotate_log_file(log_path: Path, keep: int = 20) -> Path: if log_path.parent.exists(): log_files = [x for x in log_path.parent.iterdir() if x.suffix == log_path.suffix] for log_file in log_files[::-1][keep - 1 :]: - # keep n newest files and delete the rest log_file.unlink() log_path.parent.mkdir(parents=True, exist_ok=True) @@ -129,19 +128,16 @@ def sanitize_filename(filename: str, spacer: str = ".") -> str: characters (Korean, Japanese, Chinese, etc.) instead of transliterating them to ASCII equivalents. """ - # optionally replace non-ASCII characters with ASCII equivalents if not config.unicode_filenames: filename = unidecode(filename) filename = re.sub(r"\[\(+", "[", filename) filename = re.sub(r"\)+\]", "]", filename) - - # remove or replace further characters as needed filename = "".join(c for c in filename if unicodedata.category(c) != "Mn") # hidden characters filename = filename.replace("/", " & ").replace(";", " & ") # e.g. multi-episode filenames if spacer == ".": filename = re.sub(r" - ", spacer, filename) # title separators to spacer (avoids .-. pattern) - filename = re.sub(r"[:; ]", spacer, filename) # structural chars to (spacer) - filename = re.sub(r"[\\*!?¿,'\"" "<>|$#~]", "", filename) # not filename safe chars + filename = re.sub(r"[:; ]", spacer, filename) + filename = re.sub(r"[\\*!?¿,'\"" "<>|$#~]", "", filename) filename = re.sub(rf"[{spacer}]{{2,}}", spacer, filename) # remove extra neighbouring (spacer)s filename = filename.strip(" .") # strip leading and trailing spaces and dots for OS path safety @@ -439,7 +435,6 @@ def get_country_name(code: str) -> Optional[str]: >>> get_country_name('uk') 'United Kingdom' """ - # Handle common aliases code = COUNTRY_CODE_ALIASES.get(code.lower(), code.lower()) try: @@ -470,17 +465,14 @@ def get_country_code(name: str) -> Optional[str]: 'GB' """ try: - # Try exact name match first country = pycountry.countries.get(name=name.title()) if country: return country.alpha_2.upper() - # Try common name (e.g., "Bolivia" vs "Bolivia, Plurinational State of") country = pycountry.countries.get(common_name=name.title()) if country: return country.alpha_2.upper() - # Try fuzzy search as fallback results = pycountry.countries.search_fuzzy(name) if results: return results[0].alpha_2.upper() @@ -765,25 +757,19 @@ def find_font_with_fallbacks(font_name: str, system_fonts: dict[str, Path]) -> O if not system_fonts: return None - # Strategy 1: Exact match (case-sensitive) if font_name in system_fonts: return system_fonts[font_name] - # Strategy 2: Case-insensitive match if result := find_case_insensitive(font_name, system_fonts): return result - # Strategy 3: Alias lookup if font_name in FONT_ALIASES: for alias in FONT_ALIASES[font_name]: - # Try exact match for alias if alias in system_fonts: return system_fonts[alias] - # Try case-insensitive match for alias if result := find_case_insensitive(alias, system_fonts): return result - # Strategy 4: Partial/prefix match as last resort font_name_lower = font_name.lower() for name, path in system_fonts.items(): if name.lower().startswith(font_name_lower): diff --git a/unshackle/core/utils/bitrate.py b/unshackle/core/utils/bitrate.py index 013cfe89..d725c40f 100644 --- a/unshackle/core/utils/bitrate.py +++ b/unshackle/core/utils/bitrate.py @@ -139,11 +139,9 @@ def apply_real_bitrates( for group in groups.values(): group.sort(key=lambda t: getattr(t, "bitrate", None) or 0, reverse=True) - # Initial pass: top per_group of every group, all probed concurrently. initial = [track for group in groups.values() for track in group[:per_group]] probe_batch(initial, session, log=log, workers=workers) - # Extend each group downward until unprobed tracks can't outrank probed ones. for group in groups.values(): probed = min(per_group, len(group)) while probed < len(group): diff --git a/unshackle/core/utils/click_types.py b/unshackle/core/utils/click_types.py index 2d846ddf..d8a7e3ed 100644 --- a/unshackle/core/utils/click_types.py +++ b/unshackle/core/utils/click_types.py @@ -20,7 +20,6 @@ class VideoCodecChoice(click.Choice): def __init__(self, codec_enum): self.codec_enum = codec_enum - # Build choices from both enum names and values choices = [] for codec in codec_enum: choices.append(codec.name.lower()) # e.g., "avc", "hevc" @@ -31,17 +30,14 @@ class VideoCodecChoice(click.Choice): if not value: return None - # First try to convert using the parent class converted_value = super().convert(value, param, ctx) - # Now map the converted value back to the enum for codec in self.codec_enum: if converted_value.lower() == codec.name.lower(): return codec if converted_value == codec.value: return codec - # This shouldn't happen if the parent conversion worked self.fail(f"'{value}' is not a valid video codec", param, ctx) @@ -84,17 +80,14 @@ class SubtitleCodecChoice(click.Choice): def __init__(self, codec_enum): self.codec_enum = codec_enum - # Build choices from enum names, values, and common aliases choices = [] aliases = {} for codec in codec_enum: choices.append(codec.name.lower()) # e.g., "subrip", "webvtt" - # Only add the value if it's different from common aliases value_lower = codec.value.lower() - # Add common aliases and track them if codec.name == "SubRip": if "srt" not in choices: choices.append("srt") @@ -103,32 +96,27 @@ class SubtitleCodecChoice(click.Choice): if "vtt" not in choices: choices.append("vtt") aliases["vtt"] = codec - # Also add the enum value if different if value_lower != "vtt" and value_lower not in choices: choices.append(value_lower) elif codec.name == "SubStationAlpha": if "ssa" not in choices: choices.append("ssa") aliases["ssa"] = codec - # Also add the enum value if different if value_lower != "ssa" and value_lower not in choices: choices.append(value_lower) elif codec.name == "SubStationAlphav4": if "ass" not in choices: choices.append("ass") aliases["ass"] = codec - # Also add the enum value if different if value_lower != "ass" and value_lower not in choices: choices.append(value_lower) elif codec.name == "TimedTextMarkupLang": if "ttml" not in choices: choices.append("ttml") aliases["ttml"] = codec - # Also add the enum value if different if value_lower != "ttml" and value_lower not in choices: choices.append(value_lower) else: - # For other codecs, just add the enum value if value_lower not in choices: choices.append(value_lower) @@ -144,21 +132,17 @@ class SubtitleCodecChoice(click.Choice): if str(value).lower() == "original": return "original" - # First try to convert using the parent class converted_value = super().convert(value, param, ctx) - # Check aliases first if converted_value.lower() in self.aliases: return self.aliases[converted_value.lower()] - # Now map the converted value back to the enum for codec in self.codec_enum: if converted_value.lower() == codec.name.lower(): return codec if converted_value.lower() == codec.value.lower(): return codec - # This shouldn't happen if the parent conversion worked self.fail(f"'{value}' is not a valid subtitle codec", param, ctx) @@ -460,5 +444,3 @@ LANGUAGE_RANGE = LanguageRange() QUALITY_LIST = QualityList() AUDIO_CODEC_LIST = AudioCodecList(Audio.Codec) SLOW_DELAY_RANGE = SlowDelayRange() - -# VIDEO_CODEC_CHOICE will be created dynamically when imported diff --git a/unshackle/core/utils/collections.py b/unshackle/core/utils/collections.py index 7acf8fcf..e676f1a9 100644 --- a/unshackle/core/utils/collections.py +++ b/unshackle/core/utils/collections.py @@ -57,7 +57,6 @@ def merge_dict(source: dict, destination: dict) -> None: return for key, value in source.items(): if isinstance(value, dict): - # get node or create one node = destination.setdefault(key, {}) merge_dict(value, node) else: diff --git a/unshackle/core/utils/firefox_cookie_extractor.py b/unshackle/core/utils/firefox_cookie_extractor.py index 75c00fe7..1bc70cb2 100644 --- a/unshackle/core/utils/firefox_cookie_extractor.py +++ b/unshackle/core/utils/firefox_cookie_extractor.py @@ -102,7 +102,6 @@ def get_firefox_cookies(service_settings: dict) -> Optional[CookieJar]: cookie_jar = CookieJar() - # Use a secure temporary directory with restricted permissions (Finding #5) with tempfile.TemporaryDirectory(prefix="unshackle_ff_") as tmp_dir: tmp_dir_path = Path(tmp_dir) os.chmod(tmp_dir, 0o700) @@ -131,7 +130,6 @@ def get_firefox_cookies(service_settings: dict) -> Optional[CookieJar]: rows = cursor.fetchall() for h, n, v, p, e, secure, httponly in rows: - # Final validation to ensure proper domain suffix if h == host or h.endswith(f".{host}"): c = Cookie( version=0, @@ -155,7 +153,6 @@ def get_firefox_cookies(service_settings: dict) -> Optional[CookieJar]: cookie_jar.set_cookie(c) conn.close() - # Optional LocalStorage harvesting (Finding #4) if use_local_storage: storage_data = get_local_storage_data(profile_path, priority_hosts, tmp_dir_path) for key, val in storage_data.items(): diff --git a/unshackle/core/utils/osenvironment.py b/unshackle/core/utils/osenvironment.py index 40b3eccc..215a4bf7 100644 --- a/unshackle/core/utils/osenvironment.py +++ b/unshackle/core/utils/osenvironment.py @@ -6,7 +6,6 @@ def get_os_arch(name: str) -> str: os_name = platform.system().lower() os_arch = platform.machine().lower() - # Map platform.system() output to desired OS name if os_name == "windows": os_name = "win" elif os_name == "darwin": @@ -14,11 +13,9 @@ def get_os_arch(name: str) -> str: else: os_name = "linux" - # Map platform.machine() output to desired architecture if os_arch in ["x86_64", "amd64"]: os_arch = "x64" elif os_arch == "arm64": os_arch = "arm64" - # Construct the dependency name in the desired format using the input name return f"{name}-{os_name}-{os_arch}" diff --git a/unshackle/core/utils/selector.py b/unshackle/core/utils/selector.py index 2deaf230..73fba581 100644 --- a/unshackle/core/utils/selector.py +++ b/unshackle/core/utils/selector.py @@ -49,7 +49,6 @@ class Selector: self.minimal_count = minimal_count self.dependencies = dependencies or {} - # Parent-Child mapping for quick lookup self.child_to_parent = {} for parent, children in self.dependencies.items(): for child in children: @@ -59,10 +58,8 @@ class Selector: self.selected_indices = set() self.scroll_offset = 0 - # Tree view state self.expanded_headers = set() if not collapse_on_start: - # Expand all by default self.expanded_headers.update(self.dependencies.keys()) def get_visible_indices(self) -> list[int]: @@ -72,13 +69,11 @@ class Selector: """ visible = [] for idx in range(len(self.options)): - # If it's a child, check if parent is expanded if idx in self.child_to_parent: parent = self.child_to_parent[idx] if parent in self.expanded_headers: visible.append(idx) else: - # It's a header or independent item, always visible visible.append(idx) return visible @@ -88,7 +83,6 @@ class Selector: """ visible_indices = self.get_visible_indices() - # Adjust scroll offset to ensure cursor is visible if self.cursor_index not in visible_indices: # Fallback if cursor got hidden (should be handled in move, but safety check) self.cursor_index = visible_indices[0] if visible_indices else 0 @@ -99,17 +93,14 @@ class Selector: cursor_visual_pos = 0 self.cursor_index = visible_indices[0] - # Calculate logical page start/end based on VISIBLE items start_idx = self.scroll_offset end_idx = start_idx + self.page_size - # Dynamic scroll adjustment if cursor_visual_pos < start_idx: self.scroll_offset = cursor_visual_pos elif cursor_visual_pos >= end_idx: self.scroll_offset = cursor_visual_pos - self.page_size + 1 - # Re-calc render range render_indices = visible_indices[self.scroll_offset : self.scroll_offset + self.page_size] table = Table(show_header=False, show_edge=False, box=None, pad_edge=False, padding=(0, 1, 0, 0)) @@ -130,7 +121,6 @@ class Selector: table.add_row(indicator_text, content_text) - # Fill empty rows to maintain height rows_rendered = len(render_indices) for _ in range(self.page_size - rows_rendered): table.add_row(Text(" "), Text(" ")) @@ -310,17 +300,15 @@ class Selector: if char == "\x03": return "CANCEL" - # ANSI Escape Sequences for Arrow Keys mapping = { - "\x1b[A": "UP", # Escape + [ + A - "\x1b[B": "DOWN", # Escape + [ + B - "\x1b[C": "RIGHT", # Escape + [ + C - "\x1b[D": "LEFT", # Escape + [ + D + "\x1b[A": "UP", + "\x1b[B": "DOWN", + "\x1b[C": "RIGHT", + "\x1b[D": "LEFT", } if char in mapping: return mapping[char] - # Handling manual Escape sequences if char == "\x1b": # ESC try: next1 = click.getchar() diff --git a/unshackle/core/utils/template_formatter.py b/unshackle/core/utils/template_formatter.py index a7c9be34..33c77a68 100644 --- a/unshackle/core/utils/template_formatter.py +++ b/unshackle/core/utils/template_formatter.py @@ -107,15 +107,14 @@ class TemplateFormatter: safe_value = str(value).strip() result = result.replace(placeholder, safe_value) - # Clean up multiple consecutive dots/separators and other artifacts - result = re.sub(r"\.{2,}", ".", result) # Multiple dots -> single dot + result = re.sub(r"\.{2,}", ".", result) result = re.sub(r"\.-", "-", result) # Remove dots before dashes (for dot-based templates) - result = re.sub(r"[\.\s]+([)\]])", r"\1", result) # Remove dots/spaces before closing bracket - result = re.sub(r"\(\s*\)|\[\s*\]", "", result) # Remove empty brackets (empty conditional) + result = re.sub(r"[\.\s]+([)\]])", r"\1", result) + result = re.sub(r"\(\s*\)|\[\s*\]", "", result) # bracket removal is what creates neighbouring separators, so collapse after it - result = re.sub(r"\s{2,}", " ", result) # Multiple spaces -> single space - result = re.sub(r"(?:-\s*)+-", "-", result) # Multiple dashes (with optional spaces) -> single dash - result = re.sub(r"^[\.\s\-]+|[\.\s\-]+$", "", result) # Remove leading/trailing dots, spaces, dashes + result = re.sub(r"\s{2,}", " ", result) + result = re.sub(r"(?:-\s*)+-", "-", result) + result = re.sub(r"^[\.\s\-]+|[\.\s\-]+$", "", result) result = sanitize_filename(result, spacer=self.spacer) diff --git a/unshackle/utils/base58.py b/unshackle/utils/base58.py index e63f665b..b6216585 100644 --- a/unshackle/utils/base58.py +++ b/unshackle/utils/base58.py @@ -1,23 +1,16 @@ # Clone from https://github.com/keis/base58 - """Base58 encoding Implementations of Base58 and Base58Check encodings that are compatible with the bitcoin network. """ -# This module is based upon base58 snippets found scattered over many bitcoin -# tools written in python. From what I gather the original source is from a -# forum post by Gavin Andresen, so direct your praise to him. -# This module adds shiny packaging and support for python3. - from functools import lru_cache from hashlib import sha256 from typing import Mapping, Union __version__ = "2.1.1" -# 58 character alphabet used BITCOIN_ALPHABET = b"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" RIPPLE_ALPHABET = b"rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz" XRP_ALPHABET = RIPPLE_ALPHABET diff --git a/unshackle/utils/base62.py b/unshackle/utils/base62.py index e74ddf3a..b4b4ad9e 100644 --- a/unshackle/utils/base62.py +++ b/unshackle/utils/base62.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ base62 ~~~~~~ @@ -40,7 +39,6 @@ def encodebytes(barray, charset=CHARSET_DEFAULT): check_type(barray, bytes) - # Count the number of leading zeros. leading_zeros_count = 0 for i in range(len(barray)): if barray[i] != 0: diff --git a/unshackle/utils/osenvironment.py b/unshackle/utils/osenvironment.py index 40b3eccc..215a4bf7 100644 --- a/unshackle/utils/osenvironment.py +++ b/unshackle/utils/osenvironment.py @@ -6,7 +6,6 @@ def get_os_arch(name: str) -> str: os_name = platform.system().lower() os_arch = platform.machine().lower() - # Map platform.system() output to desired OS name if os_name == "windows": os_name = "win" elif os_name == "darwin": @@ -14,11 +13,9 @@ def get_os_arch(name: str) -> str: else: os_name = "linux" - # Map platform.machine() output to desired architecture if os_arch in ["x86_64", "amd64"]: os_arch = "x64" elif os_arch == "arm64": os_arch = "arm64" - # Construct the dependency name in the desired format using the input name return f"{name}-{os_name}-{os_arch}" diff --git a/unshackle/vaults/API.py b/unshackle/vaults/API.py index bb3debae..de467eb8 100644 --- a/unshackle/vaults/API.py +++ b/unshackle/vaults/API.py @@ -133,15 +133,12 @@ class API(Vault): if error: raise error(f"{message} ({code})") - # the kid:key was new to the vault (optional) added = bool(data.get("added")) - # the key for kid was changed/updated (optional) updated = bool(data.get("updated")) return added or updated def add_keys(self, service: str, kid_keys: dict[Union[UUID, str], str]) -> int: - # Normalize keys normalized_keys = {str(kid).replace("-", ""): key for kid, key in kid_keys.items()} kid_list = list(normalized_keys.keys()) @@ -200,9 +197,7 @@ class API(Vault): if error: raise error(f"{message} ({code})") - # each kid:key that was new to the vault (optional) added = int(data.get("added", 0)) - # each key for a kid that was changed/updated (optional) updated = int(data.get("updated", 0)) total_added += added + updated diff --git a/unshackle/vaults/HTTP.py b/unshackle/vaults/HTTP.py index 0f5d2ff4..41abd826 100644 --- a/unshackle/vaults/HTTP.py +++ b/unshackle/vaults/HTTP.py @@ -201,7 +201,7 @@ class HTTP(Vault): # JSON API doesn't support getting all keys, so return empty iterator # This will cause the copy command to rely on the API's internal duplicate handling return iter([]) - else: # query mode + else: response = self.session.get( self.url, params={"service": service.lower(), "username": self.username, "password": self.password}, @@ -244,7 +244,7 @@ class HTTP(Vault): return response.get("inserted", False) except Exception: return False - else: # query mode + else: response = self.session.get( self.url, params={ @@ -295,7 +295,7 @@ class HTTP(Vault): inserted_count += 1 except Exception: continue - else: # query mode + else: for kid, key in processed_kid_keys.items(): response = self.session.get( self.url, @@ -328,7 +328,7 @@ class HTTP(Vault): yield service except Exception: return iter([]) - else: # query mode + else: response = self.session.get( self.url, params={"username": self.username, "password": self.password, "list_services": True}, @@ -393,7 +393,7 @@ class HTTP(Vault): except Exception: return InsertResult.FAILURE - else: # query mode + else: response = self.session.get( self.url, params={ diff --git a/unshackle/vaults/MySQL.py b/unshackle/vaults/MySQL.py index 7dca4749..0603d8ae 100644 --- a/unshackle/vaults/MySQL.py +++ b/unshackle/vaults/MySQL.py @@ -60,7 +60,6 @@ class MySQL(Vault): def get_keys(self, service: str) -> Iterator[tuple[str, str]]: if not self.has_table(service): - # no table, no keys, simple return None conn = self.conn_factory.get() @@ -103,7 +102,6 @@ class MySQL(Vault): (kid, key), ) if cursor.fetchone(): - # table already has this exact KID:KEY stored return True cursor.execute( # TODO: SQL injection risk diff --git a/unshackle/vaults/SQLite.py b/unshackle/vaults/SQLite.py index 0407bcc8..c0dc7823 100644 --- a/unshackle/vaults/SQLite.py +++ b/unshackle/vaults/SQLite.py @@ -25,7 +25,6 @@ class SQLite(Vault): conn = self.conn_factory.get() cursor = conn.cursor() - # Try both the original service name and lowercase version to handle case sensitivity issues service_variants = [service] if service != service.lower(): service_variants.append(service.lower()) @@ -50,7 +49,6 @@ class SQLite(Vault): def get_keys(self, service: str) -> Iterator[tuple[str, str]]: if not self.has_table(service): - # no table, no keys, simple return None conn = self.conn_factory.get() @@ -83,7 +81,6 @@ class SQLite(Vault): (kid, key), ) if cursor.fetchone(): - # table already has this exact KID:KEY stored return True cursor.execute( # TODO: SQL injection risk