diff --git a/docs/dev/rest-api/endpoints.md b/docs/dev/rest-api/endpoints.md index 05721df4..74348993 100644 --- a/docs/dev/rest-api/endpoints.md +++ b/docs/dev/rest-api/endpoints.md @@ -394,7 +394,7 @@ Create a download job. Requires `service` and `title_id`; every other field is a | `tag` | string | `null` | Release group tag. | | `tmdb_id` / `imdb_id` / `tvdb_id` / `animeapi_id` | - | `null` | External ID overrides for tagging. | | `tvdb_order` | `official`, `dvd`, `absolute`, `alternate`, `regional` | `null` | Renumber episodes to a TVDB season order. Falls back to the `tvdb_order` config option. | -| `enrich` | boolean | `false` | Override title/year from external source. | +| `enrich` | boolean | `false` | Override title/year from external source, and fill in the original language. | | `output_dir` | string | `null` | Override output directory. | | `no_cache` / `reset_cache` | boolean | `false` | Title cache controls. | diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index 4e0ec417..a6da11b7 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -154,7 +154,7 @@ Keep only certain track types, or skip certain track types. Attachments are alwa | `--imdb` | IMDb ID, e.g. `tt1375666`. | | `--tvdb` | TVDB ID (integer). Skips the series lookup that `--tvdb-order` would otherwise do. | | `--animeapi` | AnimeAPI ID, e.g. `mal:12345` or `anilist:98765` (defaults to MAL). Back-fills TMDB/IMDb/TVDB. | -| `--enrich` | Override show title/year from the external source. **Requires** one of `--tmdb`, `--imdb`, or `--animeapi`. | +| `--enrich` | Override show title/year from the external source, and fill in the original language. **Requires** one of `--tmdb`, `--imdb`, or `--animeapi`. | | `--tvdb-order` | Renumber episodes to a TVDB season order: `official` (aired), `dvd`, `absolute`, `alternate`, or `regional`. Needs `tvdb_api_key`. | #### Episode ordering diff --git a/docs/guide/downloading.md b/docs/guide/downloading.md index a8e013d8..e8228bf6 100644 --- a/docs/guide/downloading.md +++ b/docs/guide/downloading.md @@ -583,13 +583,41 @@ tagging and naming: | `--imdb` | `--imdb tt1375666` | Use this IMDb ID. | | `--tvdb` | `--tvdb 73871` | Use this TVDB ID instead of looking the series up. | | `--animeapi` | `--animeapi mal:12345` | Resolve via AnimeAPI (`mal:`/`anilist:` prefix; defaults to MAL). | -| `--enrich` | - | Override the show title and year from an external source. **Requires** one of `--tmdb`, `--imdb`, or `--animeapi`. | +| `--enrich` | - | Override the show title and year from an external source, and fill in the original language. **Requires** one of `--tmdb`, `--imdb`, or `--animeapi`. | | `--tvdb-order` | `--tvdb-order dvd` | Renumber episodes to a TVDB season order. Needs `tvdb_api_key`. | ```shell title="Force the right IMDb match and enrich the title" unshackle dl --imdb tt1375666 --enrich EXAMPLE 81234567 ``` +### What each metadata provider supplies + +Providers differ in what they answer with, so `--enrich` reads different fields depending on +which ID you gave it. A provider only runs when its key is configured: + +| Provider | Config key | Title and year | Original language | External IDs it returns | +| --- | --- | --- | --- | --- | +| TMDB | `tmdb_api_key` | yes | yes, alpha-2 such as `ko` | IMDb, TMDB, TVDB | +| TVDB | `tvdb_api_key` | yes | yes, alpha-3 such as `kor` | TMDB, TVDB, sometimes IMDb | +| OMDb | `omdb_api_key` | yes | yes, as an English name such as `Korean` | IMDb | +| IMDxAPI | `imdb_api_enabled` | yes | only when looked up by ID | IMDb | +| SIMKL | `simkl_client_id` | yes | no, it publishes a country and no language | IMDb, TMDB, TVDB | + +Whatever the tag looks like, unshackle normalises it before use, so `ko`, `kor` and `Korean` +all end up as the same language. + +`--enrich` consults these in a fixed order: + +- `--tmdb` reads TMDB. +- `--imdb` tries IMDxAPI, then falls back to OMDb. Since `imdb_api_enabled` is `false` by + default, an OMDb key is what makes this path work for most people. +- `--animeapi` supplies a title only, so pair it with `--tmdb` or `--imdb` if you also want + the year and language. + +Language is the one field `--enrich` will not overwrite. A service that already reported an +original language keeps it, because the platform knows its own catalogue better than a +third-party match does. Everything else follows the usual override behaviour. + ### Episode ordering A service does not always number a series the way TVDB's aired order does. Some services list diff --git a/docs/reference/configuration/misc.md b/docs/reference/configuration/misc.md index ce3b5c28..705a3cca 100644 --- a/docs/reference/configuration/misc.md +++ b/docs/reference/configuration/misc.md @@ -4,7 +4,9 @@ Smaller configuration areas: external metadata API keys, caching and update chec ## External API keys { #external-api-keys } -All are unset by default and enable optional metadata/geolocation features. +All are unset by default and enable optional metadata/geolocation features. For which fields +each metadata provider can actually answer with, see +[What each metadata provider supplies](../../guide/downloading.md#what-each-metadata-provider-supplies). | Key | Type | Default | Description | |-----|------|---------|-------------| diff --git a/tests/core/test_enrich_language.py b/tests/core/test_enrich_language.py new file mode 100644 index 00000000..44d5f373 --- /dev/null +++ b/tests/core/test_enrich_language.py @@ -0,0 +1,64 @@ +"""--enrich original-language enrichment.""" + +from __future__ import annotations + +import pytest + +from unshackle.commands.dl import parse_language +from unshackle.core.providers.imdbapi import primary_language as imdb_language +from unshackle.core.providers.omdb import primary_language as omdb_language +from unshackle.core.providers.tvdb import primary_language as tvdb_language + + +@pytest.mark.parametrize( + ("tag", "expected"), + [ + ("ja", "ja"), # TMDB alpha-2 + ("jpn", "ja"), # IMDb alpha-3 + ("pt-BR", "pt-BR"), + (None, None), + ("", None), + ("und", None), + ("xx-not-a-tag", None), + ], +) +def test_parse_language(tag: str | None, expected: str | None) -> None: + result = parse_language(tag) + assert (str(result) if result else None) == expected + + +def test_imdb_language_takes_the_first_spoken_language() -> None: + data = {"spokenLanguages": [{"code": "jpn", "name": "Japanese"}, {"code": "eng", "name": "English"}]} + assert imdb_language(data) == "jpn" + + +@pytest.mark.parametrize("data", [{}, {"spokenLanguages": []}, {"spokenLanguages": None}]) +def test_imdb_language_missing(data: dict) -> None: + assert imdb_language(data) is None + + +@pytest.mark.parametrize( + ("names", "expected"), + [ + ("Korean", "ko"), + ("Korean, English", "ko"), # OMDb lists the main language first + ("", None), + ("None", None), # OMDb's literal placeholder for unknown + ("Not A Language", None), + ], +) +def test_omdb_language_from_english_name(names: str, expected: str | None) -> None: + assert omdb_language({"Language": names}) == expected + + +@pytest.mark.parametrize( + ("data", "expected"), + [ + ({"primary_language": "kor"}, "kor"), # /search + ({"originalLanguage": "kor"}, "kor"), # /series/{id}/extended + ({"primary_language": ""}, None), + ({}, None), + ], +) +def test_tvdb_language_key_variants(data: dict, expected: str | None) -> None: + assert tvdb_language(data) == expected diff --git a/unshackle/commands/dl.py b/unshackle/commands/dl.py index c26dd713..1f268ec4 100644 --- a/unshackle/commands/dl.py +++ b/unshackle/commands/dl.py @@ -27,7 +27,7 @@ from uuid import UUID import click import yaml from click.core import ParameterSource -from langcodes import Language +from langcodes import Language, tag_is_valid from pymediainfo import MediaInfo from rich.console import Group from rich.padding import Padding @@ -119,6 +119,14 @@ class SkippedSubtitle(TypedDict): DL_OPTION_ALIASES = {"range": "range_", "list": "list_"} +def parse_language(tag: Optional[str]) -> Optional[Language]: + """A provider's original-language tag as a Language, or None if it is absent or not a real tag.""" + tag = (tag or "").strip() + if not tag or not tag_is_valid(tag) or tag.startswith("und"): + return None + return Language.get(tag) + + def normalize_dl_config(dl_config: dict[str, Any]) -> dict[str, Any]: """Map aliased config keys (e.g. ``range``) to their Click parameter names (``range_``).""" return {DL_OPTION_ALIASES.get(key, key): value for key, value in dl_config.items()} @@ -628,7 +636,10 @@ class dl: "--enrich", is_flag=True, default=False, - help="Override show title and year from external source. Requires --tmdb, --imdb, or --animeapi.", + help=( + "Override show title and year from external source, and fill in the original language " + "if the service did not provide one. Requires --tmdb, --imdb, or --animeapi." + ), ) @click.option( "--imdb", @@ -1513,6 +1524,7 @@ class dl: enrich_title: Optional[str] = None enrich_year: Optional[int] = None + enrich_lang: Optional[Language] = None if self.animeapi_title: enrich_title = self.animeapi_title @@ -1525,33 +1537,36 @@ class dl: enrich_year = providers.get_year_by_id( self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash ) + enrich_lang = parse_language( + providers.get_language_by_id( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + ) elif self.imdb_id: - imdbapi = providers.get_provider("imdbapi") - if imdbapi: - imdb_result = imdbapi.get_by_id(self.imdb_id, kind) + for name in ("imdbapi", "omdb"): + provider = providers.get_provider(name) + imdb_result = provider.get_by_id(self.imdb_id, kind) if provider else None if imdb_result: if not enrich_title: enrich_title = imdb_result.title enrich_year = imdb_result.year + enrich_lang = parse_language(imdb_result.original_language) + break - if enrich_title or enrich_year: - if isinstance(titles, (Series, Movies)): - for t in titles: - if enrich_title: - if isinstance(t, Episode): - t.title = enrich_title - else: - t.name = enrich_title - if enrich_year and not t.year: - t.year = enrich_year - else: + if enrich_lang: + self.log.info(f"Original language from metadata: {enrich_lang}") + + if enrich_title or enrich_year or enrich_lang: + for t in titles if isinstance(titles, (Series, Movies)) else [titles]: if enrich_title: - if isinstance(titles, Episode): - titles.title = enrich_title + if isinstance(t, Episode): + t.title = enrich_title else: - titles.name = enrich_title - if enrich_year and not titles.year: - titles.year = enrich_year + t.name = enrich_title + if enrich_year and not t.year: + t.year = enrich_year + if enrich_lang and not t.language: + t.language = enrich_lang if self.tvdb_order and isinstance(titles, Series): titles = self.apply_tvdb_order(titles, title_cacher, cache_title_id, cache_region, cache_account_hash) diff --git a/unshackle/core/providers/__init__.py b/unshackle/core/providers/__init__.py index 157196bb..95134eb7 100644 --- a/unshackle/core/providers/__init__.py +++ b/unshackle/core/providers/__init__.py @@ -196,6 +196,29 @@ def get_year_by_id( return result.year +def get_language_by_id( + tmdb_id: int, + kind: str, + title_cacher: Optional[TitleCacher] = None, + cache_title_id: Optional[str] = None, + cache_region: Optional[str] = None, + cache_account_hash: Optional[str] = None, +) -> Optional[str]: + """Get original language by TMDB ID.""" + if title_cacher and cache_title_id: + cached = title_cacher.get_cached_provider("tmdb", cache_title_id, kind, cache_region, cache_account_hash) + language = ((cached or {}).get("detail") or {}).get("original_language") + if language: + log.debug("Using cached TMDB original language: %s", language) + return language + + tmdb = get_provider("tmdb") + if not tmdb: + return None + result = tmdb.get_by_id(tmdb_id, kind) + return result.original_language if result else None + + def fetch_external_ids( tmdb_id: int, kind: str, diff --git a/unshackle/core/providers/_base.py b/unshackle/core/providers/_base.py index 62227869..77223d6f 100644 --- a/unshackle/core/providers/_base.py +++ b/unshackle/core/providers/_base.py @@ -36,6 +36,7 @@ class MetadataResult: year: Optional[int] = None kind: Optional[str] = None # "movie" or "tv" external_ids: ExternalIds = field(default_factory=ExternalIds) + original_language: Optional[str] = None # alpha-2 or alpha-3, whichever the provider speaks source: str = "" # provider name, e.g. "tmdb", "simkl", "imdbapi" raw: Optional[dict] = None # original API response for caching diff --git a/unshackle/core/providers/imdbapi.py b/unshackle/core/providers/imdbapi.py index b57bbd95..7b772a3c 100644 --- a/unshackle/core/providers/imdbapi.py +++ b/unshackle/core/providers/imdbapi.py @@ -15,6 +15,12 @@ KIND_TO_TYPES: dict[str, list[str]] = { } +def primary_language(data: dict) -> Optional[str]: + """IMDb orders spokenLanguages most prominent first, and only detail responses carry it.""" + langs = data.get("spokenLanguages") or [] + return langs[0].get("code") if langs else None + + class IMDBApiProvider(MetadataProvider): """IMDb metadata provider using IMDxAPI (api.tiffara.com, free, no API key).""" @@ -90,6 +96,7 @@ class IMDBApiProvider(MetadataProvider): year=result_year, kind=kind, external_ids=ExternalIds(imdb_id=imdb_id), + original_language=primary_language(best_match), source="imdbapi", raw=best_match, ) @@ -115,6 +122,7 @@ class IMDBApiProvider(MetadataProvider): year=result_year, kind=kind, external_ids=ExternalIds(imdb_id=data.get("id")), + original_language=primary_language(data), source="imdbapi", raw=data, ) diff --git a/unshackle/core/providers/omdb.py b/unshackle/core/providers/omdb.py index 8f450f84..a228ff42 100644 --- a/unshackle/core/providers/omdb.py +++ b/unshackle/core/providers/omdb.py @@ -4,6 +4,7 @@ from difflib import SequenceMatcher from typing import Optional, Union import requests +from langcodes import Language from unshackle.core.config import config from unshackle.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, _clean, fuzzy_match @@ -15,6 +16,17 @@ KIND_TO_TYPE: dict[str, str] = { } +def primary_language(data: dict) -> Optional[str]: + """OMDb gives English language names, most prominent first, e.g. 'Korean, English'.""" + name = (data.get("Language") or "").split(",")[0].strip() + if not name: + return None + try: + return str(Language.find(name)) + except LookupError: + return None + + def _parse_year(value: Optional[str]) -> Optional[int]: # OMDb years look like "2017", "2008–2013" or "2023–" if value and len(value) >= 4 and value[:4].isdigit(): @@ -103,6 +115,7 @@ class OMDBProvider(MetadataProvider): year=_parse_year(best_match.get("Year")), kind=kind, external_ids=ExternalIds(imdb_id=imdb_id), + original_language=primary_language(detail or best_match), source="omdb", raw=detail or best_match, ) @@ -121,6 +134,7 @@ class OMDBProvider(MetadataProvider): year=_parse_year(data.get("Year")), kind=kind, external_ids=ExternalIds(imdb_id=data.get("imdbID")), + original_language=primary_language(data), source="omdb", raw=data, ) diff --git a/unshackle/core/providers/tmdb.py b/unshackle/core/providers/tmdb.py index f50a3b16..b10011ee 100644 --- a/unshackle/core/providers/tmdb.py +++ b/unshackle/core/providers/tmdb.py @@ -90,6 +90,7 @@ class TMDBProvider(MetadataProvider): year=result_year, kind=kind, external_ids=ext, + original_language=(detail or {}).get("original_language"), source="tmdb", raw={"detail": detail or {}, "external_ids": ext_raw or {}}, ) @@ -108,6 +109,7 @@ class TMDBProvider(MetadataProvider): year=year, kind=kind, external_ids=ExternalIds(tmdb_id=int(provider_id), tmdb_kind=kind), + original_language=detail.get("original_language"), source="tmdb", raw=detail, ) diff --git a/unshackle/core/providers/tvdb.py b/unshackle/core/providers/tvdb.py index 8fa7f097..42de61e1 100644 --- a/unshackle/core/providers/tvdb.py +++ b/unshackle/core/providers/tvdb.py @@ -9,6 +9,12 @@ import requests from unshackle.core.config import config from unshackle.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, _clean, _strip_year + +def primary_language(data: dict) -> Optional[str]: + """Search results spell the key primary_language, the extended record originalLanguage.""" + return data.get("primary_language") or data.get("originalLanguage") or None + + KIND_TO_TYPE: dict[str, str] = {"movie": "movie", "tv": "series"} KIND_TO_PATH: dict[str, str] = {"movie": "movies", "tv": "series"} @@ -153,6 +159,7 @@ class TVDBProvider(MetadataProvider): year=_parse_int(best.get("year")), kind=kind, external_ids=ext, + original_language=primary_language(best), source="tvdb", raw=best, ) @@ -176,6 +183,7 @@ class TVDBProvider(MetadataProvider): year=_parse_int(detail.get("year")), kind=kind, external_ids=ext, + original_language=primary_language(detail), source="tvdb", raw=detail, )