mirror of
https://github.com/unshackle-dl/unshackle.git
synced 2026-11-12 12:05:07 +00:00
feat(dl): add --sample and --include-sample
--sample downloads only a time window of a title (default 120-180 s) and cuts the mux to it. --include-sample cuts a 60 s sample from a finished file. Both write to a Sample folder with a SAMPLE tag on the filename
This commit is contained in:
@@ -462,6 +462,8 @@ Make a download job. It requires `service` and `title_id`. Every other field is
|
|||||||
| `cdm_only` | boolean | `null` | Force CDM-only (`true`) or vault-only (`false`) key retrieval. |
|
| `cdm_only` | boolean | `null` | Force CDM-only (`true`) or vault-only (`false`) key retrieval. |
|
||||||
| `proxy` / `no_proxy` / `no_proxy_download` / `proxy_download` | string / bool / bool / string | `null` / `false` / `false` / `null` | Proxy controls. `proxy` and `proxy_download` must be full proxy URIs unless the API key has `server_proxy`. A Control D resolver, `controld://<resolver>@dns.controld.com`, counts as a full proxy URI: the server runs a forwarder for it. |
|
| `proxy` / `no_proxy` / `no_proxy_download` / `proxy_download` | string / bool / bool / string | `null` / `false` / `false` / `null` | Proxy controls. `proxy` and `proxy_download` must be full proxy URIs unless the API key has `server_proxy`. A Control D resolver, `controld://<resolver>@dns.controld.com`, counts as a full proxy URI: the server runs a forwarder for it. |
|
||||||
| `no_folder` / `no_source` / `no_mux` | boolean | `false` | Output/mux controls. |
|
| `no_folder` / `no_source` / `no_mux` | boolean | `false` | Output/mux controls. |
|
||||||
|
| `sample` | boolean/string | `null` | Download a short sample and write it with a `SAMPLE` tag on the filename in a `Sample` folder inside the output folder. `true` is 60 seconds from 2 minutes in; `"0-60"` is a window in seconds. |
|
||||||
|
| `include_sample` | boolean | `false` | Download the full title, then also cut a 60 second sample from each muxed file into a `Sample` folder next to it. |
|
||||||
| `workers` | int | `null` | Threads per track. |
|
| `workers` | int | `null` | Threads per track. |
|
||||||
| `adaptive_workers` | boolean | `false` | Scale segment workers to measured CDN throughput, up to the `workers` cap. |
|
| `adaptive_workers` | boolean | `false` | Scale segment workers to measured CDN throughput, up to the `workers` cap. |
|
||||||
| `download_processes` | int | `1` | Split a track's segments across this many processes. Only engages for large batches. |
|
| `download_processes` | int | `1` | Split a track's segments across this many processes. Only engages for large batches. |
|
||||||
|
|||||||
@@ -153,6 +153,8 @@ Keep only certain track types, or skip certain track types.
|
|||||||
| `--no-folder` | Disable folder creation for TV shows. |
|
| `--no-folder` | Disable folder creation for TV shows. |
|
||||||
| `--no-source` | Remove the source tag from the filename/path. |
|
| `--no-source` | Remove the source tag from the filename/path. |
|
||||||
| `--no-mux` | Do not mux; keep individual track files. |
|
| `--no-mux` | Do not mux; keep individual track files. |
|
||||||
|
| `--sample` | Download a short sample and write it with a `SAMPLE` tag on the filename in a `Sample` folder inside the output folder. Bare `--sample` = 60 seconds from 2 minutes in; `--sample 0-60` = a custom window in seconds. |
|
||||||
|
| `--include-sample` | Download the full title, then also cut a 60 second sample from each muxed file into a `Sample` folder next to it. |
|
||||||
| `--tag` | Group tag override. |
|
| `--tag` | Group tag override. |
|
||||||
| `--repack` | Add a `REPACK` tag to the filename. |
|
| `--repack` | Add a `REPACK` tag to the filename. |
|
||||||
| `--postscript` | Run a command after each output file, with `{variable}` placeholders substituted. Repeatable. Replaces the `post_scripts` config for this run. See [Post-scripts](../reference/configuration/post-scripts.md). |
|
| `--postscript` | Run a command after each output file, with `{variable}` placeholders substituted. Repeatable. Replaces the `post_scripts` config for this run. See [Post-scripts](../reference/configuration/post-scripts.md). |
|
||||||
|
|||||||
@@ -711,9 +711,23 @@ By default, `mkvmerge` muxes the tracks of a movie or an episode into a single M
|
|||||||
| Flag | Behaviour | Default source |
|
| Flag | Behaviour | Default source |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `--no-mux` | Do not mux; keep the individual track files. | - |
|
| `--no-mux` | Do not mux; keep the individual track files. | - |
|
||||||
|
| `--sample` | Download a short sample: 60 seconds from 2 minutes in, or `--sample START-END` for a window in seconds. | - |
|
||||||
|
| `--include-sample` | Download the full title, then also cut a 60 second sample from each muxed file into a `Sample` folder. | - |
|
||||||
| `--split-audio` | Write a separate output file per audio codec instead of merging all audio. | config `muxing.merge_audio` (on) |
|
| `--split-audio` | Write a separate output file per audio codec instead of merging all audio. | config `muxing.merge_audio` (on) |
|
||||||
| `--merge-video` | Mux video tracks that share a height, range, and codec into one file, so only language varies inside a file. | config `muxing.merge_video` (off) |
|
| `--merge-video` | Mux video tracks that share a height, range, and codec into one file, so only language varies inside a file. | config `muxing.merge_video` (off) |
|
||||||
|
|
||||||
|
!!! note "`--sample` needs a segmented track"
|
||||||
|
`--sample` downloads only the segments that hold the window, and `mkvmerge` cuts the output
|
||||||
|
to the window. The cut starts at the first video keyframe in the window, so a sample can be
|
||||||
|
some seconds short. unshackle downloads each subtitle track in full and then keeps only the
|
||||||
|
cues of the window.
|
||||||
|
|
||||||
|
The run stops before the download starts in two conditions: a service gives a video or
|
||||||
|
audio track as one file with no segments and no segment index, or the title ends before the
|
||||||
|
window starts.
|
||||||
|
|
||||||
|
A sidecar subtitle file with `sidecar_format: original` stays whole.
|
||||||
|
|
||||||
!!! note "Music is never muxed"
|
!!! note "Music is never muxed"
|
||||||
A song skips the muxer. unshackle keeps the container that the service
|
A song skips the muxer. unshackle keeps the container that the service
|
||||||
delivered, such as `.flac`, `.m4a`, or `.mp3`, and writes the metadata into that file.
|
delivered, such as `.flac`, `.m4a`, or `.mp3`, and writes the metadata into that file.
|
||||||
|
|||||||
@@ -330,6 +330,22 @@ A few muxing behaviors are automatic and not configurable:
|
|||||||
|
|
||||||
The `--no-mux` flag skips muxing entirely and writes the individual track files, each with a track-type suffix, into the same output/folder structure.
|
The `--no-mux` flag skips muxing entirely and writes the individual track files, each with a track-type suffix, into the same output/folder structure.
|
||||||
|
|
||||||
|
## Sample files
|
||||||
|
|
||||||
|
[`--sample`](cli-reference.md#output-muxing-files) and `--include-sample` write a sample into a
|
||||||
|
`Sample` folder inside the output folder of the title. The sample keeps the filename of the
|
||||||
|
title, with a `SAMPLE` tag at the end. The tag uses the separator of the output template.
|
||||||
|
|
||||||
|
```text title="A movie with --include-sample"
|
||||||
|
Some.Movie.2026.1080p.EXAMPLE.WEB-DL.H.264-TAG/
|
||||||
|
├── Some.Movie.2026.1080p.EXAMPLE.WEB-DL.H.264-TAG.mkv
|
||||||
|
└── Sample/
|
||||||
|
└── Some.Movie.2026.1080p.EXAMPLE.WEB-DL.H.264-TAG.SAMPLE.mkv
|
||||||
|
```
|
||||||
|
|
||||||
|
With `--sample`, the `Sample` folder holds the only output file. If the run makes no title
|
||||||
|
folder, the `Sample` folder is in the downloads directory.
|
||||||
|
|
||||||
## Tags & group naming
|
## Tags & group naming
|
||||||
|
|
||||||
The release-group tag is the `-TAG` portion at the end of scene-style names, which the `{tag}` variable gives.
|
The release-group tag is the `-TAG` portion at the end of scene-style names, which the `{tag}` variable gives.
|
||||||
|
|||||||
@@ -0,0 +1,456 @@
|
|||||||
|
"""A ``--sample`` run downloads only the segments of its window and cuts the mux to the window."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import functools
|
||||||
|
import http.server
|
||||||
|
import struct
|
||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from typing import Optional
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import click
|
||||||
|
import pytest
|
||||||
|
from requests import Session
|
||||||
|
|
||||||
|
from unshackle.commands.dl import sample_name
|
||||||
|
from unshackle.core import binaries
|
||||||
|
from unshackle.core.api.handlers import validate_download_parameters
|
||||||
|
from unshackle.core.config import config
|
||||||
|
from unshackle.core.manifests.dash import DASH
|
||||||
|
from unshackle.core.services import Services
|
||||||
|
from unshackle.core.titles import Episode, Movie
|
||||||
|
from unshackle.core.tracks import Audio, Chapter, Subtitle, Tracks, Video
|
||||||
|
from unshackle.core.tracks.track import parse_sidx
|
||||||
|
from unshackle.core.utils.click_types import SAMPLE_RANGE
|
||||||
|
from unshackle.core.utils.xml import load_xml
|
||||||
|
|
||||||
|
STARTS = [1000 + i * 4000 for i in range(100)] # 4 s segments at timescale 1000
|
||||||
|
|
||||||
|
|
||||||
|
def make_video() -> Video:
|
||||||
|
return Video(id_="v", url="https://x/v", codec=Video.Codec.AVC, language="en", width=1920, height=1080, fps=24)
|
||||||
|
|
||||||
|
|
||||||
|
def make_audio(url: str = "https://x/a") -> Audio:
|
||||||
|
return Audio(id_="a", url=url, codec=Audio.Codec.AAC, language="en", bitrate=128000)
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_window_keeps_every_segment() -> None:
|
||||||
|
assert make_audio().sample_slice(STARTS, 1000) == slice(None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_window_is_measured_from_the_first_segment() -> None:
|
||||||
|
track = make_audio()
|
||||||
|
assert track.sample_slice(STARTS, 1000, (62, 120)) == slice(15, 30)
|
||||||
|
assert track.sample_start == 60
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_video_keeps_one_segment_after_the_window() -> None:
|
||||||
|
"""mkvmerge ends the cut only at a video keyframe, so the video must reach past the window."""
|
||||||
|
assert make_video().sample_slice(STARTS, 1000, (0, 60)) == slice(0, 16)
|
||||||
|
assert make_audio().sample_slice(STARTS, 1000, (0, 60)) == slice(0, 15)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_subtitle_keeps_every_segment() -> None:
|
||||||
|
subtitle = Subtitle(id_="s", url="https://x/s", codec=Subtitle.Codec.WebVTT, language="en")
|
||||||
|
assert subtitle.sample_slice(STARTS, 1000, (62, 120)) == slice(None)
|
||||||
|
assert subtitle.sample_start == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_window_after_the_end_stops_the_run() -> None:
|
||||||
|
with pytest.raises(click.ClickException, match="ends before the sample window"):
|
||||||
|
make_audio().sample_slice(STARTS[:10], 1000, (120, 180))
|
||||||
|
|
||||||
|
|
||||||
|
TIMELINE_MPD = """
|
||||||
|
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||||
|
<Period id="0" start="PT0S">
|
||||||
|
<AdaptationSet mimeType="audio/mp4">
|
||||||
|
<SegmentTemplate timescale="1" media="a/$Number$-$Time$.mp4" startNumber="5">
|
||||||
|
<SegmentTimeline><S t="1000" d="4" r="99"/></SegmentTimeline>
|
||||||
|
</SegmentTemplate>
|
||||||
|
<Representation id="a" bandwidth="1"/>
|
||||||
|
</AdaptationSet>
|
||||||
|
</Period>
|
||||||
|
</MPD>
|
||||||
|
"""
|
||||||
|
|
||||||
|
DURATION_MPD = """
|
||||||
|
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||||
|
<Period id="0" start="PT0S" duration="PT400S">
|
||||||
|
<AdaptationSet mimeType="audio/mp4">
|
||||||
|
<SegmentTemplate timescale="25" duration="100" media="a/$Number$.mp4" startNumber="5"/>
|
||||||
|
<Representation id="a" bandwidth="1"/>
|
||||||
|
</AdaptationSet>
|
||||||
|
</Period>
|
||||||
|
</MPD>
|
||||||
|
"""
|
||||||
|
|
||||||
|
LIST_MPD = """
|
||||||
|
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||||
|
<Period id="0" start="PT0S">
|
||||||
|
<AdaptationSet mimeType="audio/mp4">
|
||||||
|
<Representation id="a" bandwidth="1">
|
||||||
|
<BaseURL>https://example.com/a/</BaseURL>
|
||||||
|
<SegmentList timescale="1" duration="4">{urls}</SegmentList>
|
||||||
|
</Representation>
|
||||||
|
</AdaptationSet>
|
||||||
|
</Period>
|
||||||
|
</MPD>
|
||||||
|
""".format(urls="".join(f'<SegmentURL media="{n}.mp4"/>' for n in range(100)))
|
||||||
|
|
||||||
|
BASE_MPD = """
|
||||||
|
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||||
|
<Period id="0" start="PT0S">
|
||||||
|
<AdaptationSet mimeType="audio/mp4">
|
||||||
|
<Representation id="a" bandwidth="1"><BaseURL>https://example.com/a.mp4</BaseURL></Representation>
|
||||||
|
</AdaptationSet>
|
||||||
|
</Period>
|
||||||
|
</MPD>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
WINDOW = (62, 120)
|
||||||
|
|
||||||
|
|
||||||
|
def dash_names(mpd: str, window: Optional[tuple[int, int]]) -> tuple[list[str], float]:
|
||||||
|
tree = load_xml(mpd.encode())
|
||||||
|
period = tree.find("Period")
|
||||||
|
aset = period.find("AdaptationSet")
|
||||||
|
rep = aset.find("Representation")
|
||||||
|
track = make_audio()
|
||||||
|
_, segments, _, _, _ = DASH.get_period_segments(
|
||||||
|
period, aset, rep, tree, track, "https://example.com/main.mpd", MagicMock(), window=window
|
||||||
|
)
|
||||||
|
return [url.rsplit("/", 1)[1].split(".")[0] for url, _ in segments], track.sample_start
|
||||||
|
|
||||||
|
|
||||||
|
def test_dash_timeline_keeps_the_numbers_and_times_of_the_window() -> None:
|
||||||
|
names, start = dash_names(TIMELINE_MPD, WINDOW)
|
||||||
|
assert names == [f"{5 + n}-{1000 + n * 4}" for n in range(15, 30)]
|
||||||
|
assert start == 60
|
||||||
|
|
||||||
|
|
||||||
|
def test_dash_fixed_duration_keeps_the_numbers_of_the_window() -> None:
|
||||||
|
names, start = dash_names(DURATION_MPD, WINDOW)
|
||||||
|
assert names == [str(5 + n) for n in range(15, 30)]
|
||||||
|
assert start == 60
|
||||||
|
|
||||||
|
|
||||||
|
def test_dash_segment_list_keeps_the_urls_of_the_window() -> None:
|
||||||
|
names, start = dash_names(LIST_MPD, WINDOW)
|
||||||
|
assert names == [str(n) for n in range(15, 30)]
|
||||||
|
assert start == 60
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mpd", [TIMELINE_MPD, DURATION_MPD, LIST_MPD])
|
||||||
|
def test_a_caller_that_passes_no_window_gets_every_segment(mpd: str, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""The bitrate probe divides by the full duration, so a sample run must not cut its segment list."""
|
||||||
|
monkeypatch.setattr(config, "sample", WINDOW)
|
||||||
|
assert len(dash_names(mpd, None)[0]) == 100
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("mpd", "timed"), [(TIMELINE_MPD, True), (LIST_MPD, True), (BASE_MPD, False)])
|
||||||
|
def test_a_track_given_as_one_file_has_no_segment_timing(mpd: str, timed: bool) -> None:
|
||||||
|
aset = load_xml(mpd.encode()).find("Period").find("AdaptationSet")
|
||||||
|
track = SimpleNamespace(data={"dash": {"adaptation_set": aset, "representation": aset.find("Representation")}})
|
||||||
|
assert DASH.has_segment_timing(track) is timed
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def served(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> str:
|
||||||
|
handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=str(tmp_path))
|
||||||
|
handler.log_message = lambda *args: None # type: ignore[attr-defined]
|
||||||
|
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||||
|
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||||
|
(tmp_path / "temp").mkdir()
|
||||||
|
monkeypatch.setattr(config.directories, "temp", tmp_path / "temp")
|
||||||
|
yield f"http://127.0.0.1:{server.server_port}"
|
||||||
|
server.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_hls_subtitle_downloads_whole_in_a_sample_run(
|
||||||
|
tmp_path: Path, served: str, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
for n in range(4):
|
||||||
|
(tmp_path / f"s{n}.vtt").write_text(
|
||||||
|
f"WEBVTT\n\n00:00:{n * 10 + 1:02}.000 --> 00:00:{n * 10 + 2:02}.000\ncue {n}\n"
|
||||||
|
)
|
||||||
|
(tmp_path / "s.m3u8").write_text(
|
||||||
|
"#EXTM3U\n#EXT-X-TARGETDURATION:10\n"
|
||||||
|
+ "".join(f"#EXTINF:10.0,\ns{n}.vtt\n" for n in range(4))
|
||||||
|
+ "#EXT-X-ENDLIST\n"
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(config, "sample", (12, 18))
|
||||||
|
subtitle = Subtitle(
|
||||||
|
id_="s", url=f"{served}/s.m3u8", codec=Subtitle.Codec.WebVTT, language="en", descriptor=Subtitle.Descriptor.HLS
|
||||||
|
)
|
||||||
|
|
||||||
|
subtitle.download(session=Session(), prepare_drm=lambda *a, **k: None, progress=lambda **k: None)
|
||||||
|
|
||||||
|
text = subtitle.path.read_text()
|
||||||
|
assert all(f"cue {n}" in text for n in range(4))
|
||||||
|
|
||||||
|
|
||||||
|
def test_subtitle_trim_keeps_the_cues_of_the_window_from_zero(tmp_path: Path) -> None:
|
||||||
|
subtitle = Subtitle(id_="s", url="https://x/s", codec=Subtitle.Codec.SubRip, language="en")
|
||||||
|
subtitle.path = tmp_path / "s.srt"
|
||||||
|
subtitle.path.write_text(
|
||||||
|
"1\n00:00:05,000 --> 00:00:06,000\nbefore\n\n"
|
||||||
|
"2\n00:02:10,000 --> 00:02:12,000\ninside\n\n"
|
||||||
|
"3\n00:10:00,000 --> 00:10:01,000\nafter\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert subtitle.trim(120, 180) is True
|
||||||
|
|
||||||
|
text = subtitle.path.read_text()
|
||||||
|
assert "inside" in text and "before" not in text and "after" not in text
|
||||||
|
assert "00:00:10,000 --> 00:00:12,000" in text
|
||||||
|
assert subtitle.sample_start == 120
|
||||||
|
|
||||||
|
|
||||||
|
def test_subtitle_trim_leaves_an_unreadable_file_whole(tmp_path: Path) -> None:
|
||||||
|
subtitle = Subtitle(id_="s", url="https://x/s", codec=Subtitle.Codec.fTTML, language="en")
|
||||||
|
subtitle.path = tmp_path / "s.stpp"
|
||||||
|
subtitle.path.write_bytes(b"\x00\x01not a subtitle")
|
||||||
|
|
||||||
|
assert subtitle.trim(120, 180) is False
|
||||||
|
assert subtitle.path.read_bytes() == b"\x00\x01not a subtitle"
|
||||||
|
|
||||||
|
|
||||||
|
def mux_command(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> list[str]:
|
||||||
|
class Stop(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
captured: list[list[str]] = []
|
||||||
|
|
||||||
|
def fake_popen(args: list[str], **kwargs: object) -> None:
|
||||||
|
captured.append(args)
|
||||||
|
raise Stop
|
||||||
|
|
||||||
|
monkeypatch.setattr("unshackle.core.tracks.tracks.subprocess.Popen", fake_popen)
|
||||||
|
monkeypatch.setattr(binaries, "MKVToolNix", Path("mkvmerge"))
|
||||||
|
|
||||||
|
video, audio = make_video(), make_audio()
|
||||||
|
trimmed = Subtitle(id_="s1", url="https://x/s1", codec=Subtitle.Codec.SubRip, language="en")
|
||||||
|
whole = Subtitle(id_="s2", url="https://x/s2", codec=Subtitle.Codec.SubRip, language="fr")
|
||||||
|
for track, name in ((video, "v.mp4"), (audio, "a.mp4"), (trimmed, "s1.srt"), (whole, "s2.srt")):
|
||||||
|
track.path = tmp_path / name
|
||||||
|
track.path.write_bytes(b"\x00" * 16)
|
||||||
|
video.sample_start, audio.sample_start, trimmed.sample_start = 120.0, 119.5, 121.0
|
||||||
|
tracks = Tracks([video, audio, trimmed, whole])
|
||||||
|
tracks.chapters.add(Chapter("00:02:05.000", "Scene"))
|
||||||
|
|
||||||
|
with pytest.raises(Stop):
|
||||||
|
tracks.mux("Title", delete=False, progress=lambda **k: None, output_path=tmp_path / "out.mkv")
|
||||||
|
return captured[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_mux_shifts_each_track_and_cuts_to_the_window(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(config, "sample", (121, 181))
|
||||||
|
command = mux_command(tmp_path, monkeypatch)
|
||||||
|
|
||||||
|
def sync_before(name: str) -> str:
|
||||||
|
file_at = command.index(str(tmp_path / name))
|
||||||
|
options = command[:file_at]
|
||||||
|
sync_at = len(options) - 1 - options[::-1].index("--sync")
|
||||||
|
assert all(not arg.endswith((".mp4", ".srt")) for arg in command[sync_at:file_at]), (
|
||||||
|
"the --sync is another file's"
|
||||||
|
)
|
||||||
|
return command[sync_at + 1]
|
||||||
|
|
||||||
|
assert command[command.index("--split") + 1] == "parts:1.500s-61.500s"
|
||||||
|
assert sync_before("v.mp4") == "0:500"
|
||||||
|
assert "--sync" not in command[: command.index(str(tmp_path / "a.mp4"))][-12:], "the earliest track is not moved"
|
||||||
|
assert sync_before("s1.srt") == "0:1500"
|
||||||
|
assert sync_before("s2.srt") == "0:-119500"
|
||||||
|
assert command[command.index("--chapter-sync") + 1] == "-119500"
|
||||||
|
assert command.index("--chapter-sync") < command.index("--chapters")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_full_download_adds_no_sample_options(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(config, "sample", None)
|
||||||
|
command = mux_command(tmp_path, monkeypatch)
|
||||||
|
assert not {"--split", "--sync", "--chapter-sync"} & set(command)
|
||||||
|
|
||||||
|
|
||||||
|
def test_sample_range_reads_a_window() -> None:
|
||||||
|
assert SAMPLE_RANGE.convert("60-120", None, None) == (60, 120)
|
||||||
|
assert SAMPLE_RANGE.convert(True, None, None) == (120, 180)
|
||||||
|
assert SAMPLE_RANGE.convert(False, None, None) is None
|
||||||
|
for bad in ("120-60", "60-60", "abc", "-5-10", "60"):
|
||||||
|
with pytest.raises(click.BadParameter):
|
||||||
|
SAMPLE_RANGE.convert(bad, None, None)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("sample", [True, "60-120", [60, 120]])
|
||||||
|
def test_the_api_accepts_a_sample_window(sample: object) -> None:
|
||||||
|
assert validate_download_parameters({"sample": sample}) is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("sample", ["abc", "120-60", [1, 2, 3], ["a", "b"], 5, {"start": 1}])
|
||||||
|
def test_the_api_refuses_a_bad_sample_window(sample: object) -> None:
|
||||||
|
assert "sample" in (validate_download_parameters({"sample": sample}) or "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_api_refuses_sample_with_include_sample() -> None:
|
||||||
|
assert "include_sample" in (validate_download_parameters({"sample": True, "include_sample": True}) or "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_sample_name_uses_the_separator_of_the_template(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
separators = {"movies": ".", "series": " "}
|
||||||
|
monkeypatch.setattr(config, "get_template_separator", lambda kind="movies": separators[kind])
|
||||||
|
movie = Movie.__new__(Movie)
|
||||||
|
episode = Episode.__new__(Episode)
|
||||||
|
assert sample_name("Some.Movie.2026.1080p-TAG", movie) == "Some.Movie.2026.1080p-TAG.SAMPLE"
|
||||||
|
assert sample_name("Some Show S01E01 1080p-TAG", episode) == "Some Show S01E01 1080p-TAG SAMPLE"
|
||||||
|
|
||||||
|
|
||||||
|
SIDX_MPD = """
|
||||||
|
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" mediaPresentationDuration="PT400S">
|
||||||
|
<Period id="0" start="PT0S">
|
||||||
|
<AdaptationSet mimeType="audio/mp4">
|
||||||
|
<Representation id="a" bandwidth="1">
|
||||||
|
<BaseURL>https://example.com/a.mp4</BaseURL>
|
||||||
|
<SegmentBase indexRange="800-{last}"><Initialization range="0-799"/></SegmentBase>
|
||||||
|
</Representation>
|
||||||
|
</AdaptationSet>
|
||||||
|
</Period>
|
||||||
|
</MPD>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def sidx_box(count: int = 100, size: int = 5000, duration: int = 4000, timescale: int = 1000) -> bytes:
|
||||||
|
body = struct.pack(">IIIII", 0, 1, timescale, 0, 0) + struct.pack(">HH", 0, count)
|
||||||
|
body += b"".join(struct.pack(">III", size, duration, 0x90000000) for _ in range(count))
|
||||||
|
return struct.pack(">I", 8 + len(body)) + b"sidx" + body
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_sidx_gives_the_time_and_byte_range_of_each_subsegment() -> None:
|
||||||
|
box = sidx_box(count=3)
|
||||||
|
end = 800 + len(box)
|
||||||
|
assert parse_sidx(box, 800) == (
|
||||||
|
1000,
|
||||||
|
[(0, end, end + 4999), (4000, end + 5000, end + 9999), (8000, end + 10000, end + 14999)],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_sidx_refuses_what_it_cannot_cut() -> None:
|
||||||
|
index_of_indexes = bytearray(sidx_box(count=2))
|
||||||
|
index_of_indexes[32] |= 0x80
|
||||||
|
assert parse_sidx(bytes(index_of_indexes), 0) == (0, [])
|
||||||
|
assert parse_sidx(b"\x00\x00\x00\x20moof" + b"\x00" * 24, 0) == (0, [])
|
||||||
|
assert parse_sidx(sidx_box(count=3)[:-4], 0) == (0, [])
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_single_file_dash_track_is_cut_by_its_segment_index(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
box = sidx_box()
|
||||||
|
requested: list[str] = []
|
||||||
|
|
||||||
|
def get(url: str, headers: dict[str, str] | None = None) -> MagicMock:
|
||||||
|
byte_range = (headers or {}).get("Range", "")
|
||||||
|
requested.append(byte_range)
|
||||||
|
content = box if byte_range == f"bytes=800-{800 + len(box) - 1}" else b"\x00" * 800
|
||||||
|
return MagicMock(content=content, headers={"Content-Range": "bytes 0-799/900000"})
|
||||||
|
|
||||||
|
tree = load_xml(SIDX_MPD.format(last=800 + len(box) - 1).encode())
|
||||||
|
period = tree.find("Period")
|
||||||
|
aset = period.find("AdaptationSet")
|
||||||
|
rep = aset.find("Representation")
|
||||||
|
track = make_audio()
|
||||||
|
track.data["dash"] = {"adaptation_set": aset, "representation": rep}
|
||||||
|
session = MagicMock(get=get)
|
||||||
|
first_media_byte = 800 + len(box)
|
||||||
|
|
||||||
|
_, segments, timescale, starts, _ = DASH.get_period_segments(
|
||||||
|
period, aset, rep, tree, track, "https://example.com/main.mpd", session, probe_kid=False, window=WINDOW
|
||||||
|
)
|
||||||
|
|
||||||
|
assert DASH.has_segment_timing(track) is True
|
||||||
|
assert timescale == 1000 and starts == [n * 4000 for n in range(15, 30)]
|
||||||
|
assert segments[0] == (
|
||||||
|
"https://example.com/a.mp4",
|
||||||
|
f"{first_media_byte + 15 * 5000}-{first_media_byte + 16 * 5000 - 1}",
|
||||||
|
)
|
||||||
|
assert len(segments) == 15 and track.sample_start == 60
|
||||||
|
|
||||||
|
requested.clear()
|
||||||
|
_, whole, _, _, _ = DASH.get_period_segments(
|
||||||
|
period, aset, rep, tree, track, "https://example.com/main.mpd", session
|
||||||
|
)
|
||||||
|
assert len(whole) == 1 and not any("800-" in r for r in requested), "a full download does not read the index"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("args", "window"),
|
||||||
|
[
|
||||||
|
(["--sample", "TAG", "-t", "x", "id"], (120, 180)),
|
||||||
|
(["--sample", "60-120", "TAG", "id"], (60, 120)),
|
||||||
|
(["TAG", "id"], None),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_a_bare_sample_option_does_not_take_the_service_tag(args: list[str], window: object) -> None:
|
||||||
|
seen: dict[str, object] = {}
|
||||||
|
|
||||||
|
class Group(Services):
|
||||||
|
def list_commands(self, ctx: click.Context) -> list[str]:
|
||||||
|
return ["TAG"]
|
||||||
|
|
||||||
|
def get_command(self, ctx: click.Context, name: str) -> click.Command:
|
||||||
|
return click.Command(
|
||||||
|
name, params=[click.Option(["-t"]), click.Argument(["title"])], callback=lambda **k: None
|
||||||
|
)
|
||||||
|
|
||||||
|
@click.group(cls=Group)
|
||||||
|
@click.option("--sample", type=SAMPLE_RANGE, default=None)
|
||||||
|
def group(sample: object) -> None:
|
||||||
|
seen["sample"] = sample
|
||||||
|
|
||||||
|
group(args, standalone_mode=False)
|
||||||
|
assert seen["sample"] == window
|
||||||
|
|
||||||
|
|
||||||
|
class RangeSession:
|
||||||
|
"""Serves byte ranges of one in-memory file and records each range it was asked for."""
|
||||||
|
|
||||||
|
def __init__(self, data: bytes) -> None:
|
||||||
|
self.data = data
|
||||||
|
self.ranges: list[str] = []
|
||||||
|
|
||||||
|
def get(self, url: str, headers: dict[str, str]) -> SimpleNamespace:
|
||||||
|
self.ranges.append(headers["Range"])
|
||||||
|
first, last = (int(n) for n in headers["Range"].removeprefix("bytes=").split("-"))
|
||||||
|
return SimpleNamespace(ok=True, content=self.data[first : last + 1])
|
||||||
|
|
||||||
|
|
||||||
|
def one_file(moov_size: int, index: bytes) -> bytes:
|
||||||
|
ftyp = struct.pack(">I4s", 16, b"ftyp") + b"isom" + b"\x00" * 4
|
||||||
|
moov = struct.pack(">I4s", moov_size, b"moov") + b"\x00" * (moov_size - 8)
|
||||||
|
return ftyp + moov + index + b"\x00" * 600_000
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("moov_size", [700, 200_000])
|
||||||
|
def test_a_one_file_track_is_cut_by_the_index_in_the_file(moov_size: int, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""A large moov puts the index after the first probe, so the probe must read on."""
|
||||||
|
monkeypatch.setattr(config, "sample", WINDOW)
|
||||||
|
box = sidx_box()
|
||||||
|
session = RangeSession(one_file(moov_size, box))
|
||||||
|
audio = make_audio("https://x/a.mp4")
|
||||||
|
|
||||||
|
init, byte_range = audio.sample_byte_range(session)
|
||||||
|
|
||||||
|
media = 16 + moov_size + len(box)
|
||||||
|
assert init == session.data[: 16 + moov_size], "the init data stops before the index"
|
||||||
|
assert byte_range == f"{media + 15 * 5000}-{media + 30 * 5000 - 1}"
|
||||||
|
assert audio.sample_start == 60
|
||||||
|
assert all(r.startswith("bytes=") for r in session.ranges), "every request is a byte range"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_one_file_track_with_no_index_cannot_be_cut(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(config, "sample", WINDOW)
|
||||||
|
mdat = struct.pack(">I4s", 600_000, b"mdat") + b"\x00" * (600_000 - 8)
|
||||||
|
audio = make_audio("https://x/a.mp4")
|
||||||
|
assert audio.sample_byte_range(RangeSession(one_file(700, b"")[:716] + mdat)) is None
|
||||||
+131
-5
@@ -46,6 +46,7 @@ from unshackle.core.cdm.detect import cdm_type_stub, is_playready_cdm, is_remote
|
|||||||
from unshackle.core.config import cdm_entry_names_device, config, resolve_cdm_name, resolve_decryption
|
from unshackle.core.config import cdm_entry_names_device, config, resolve_cdm_name, resolve_decryption
|
||||||
from unshackle.core.console import GradientPulseBarColumn, SyncLive, console, listing_panel
|
from unshackle.core.console import GradientPulseBarColumn, SyncLive, console, listing_panel
|
||||||
from unshackle.core.constants import (
|
from unshackle.core.constants import (
|
||||||
|
DEFAULT_SAMPLE,
|
||||||
DOWNLOAD_ALL_DRM,
|
DOWNLOAD_ALL_DRM,
|
||||||
DOWNLOAD_CANCELLED,
|
DOWNLOAD_CANCELLED,
|
||||||
DOWNLOAD_LICENCE_ONLY,
|
DOWNLOAD_LICENCE_ONLY,
|
||||||
@@ -107,6 +108,7 @@ from unshackle.core.utils.click_types import (
|
|||||||
AUDIO_CODEC_LIST,
|
AUDIO_CODEC_LIST,
|
||||||
LANGUAGE_RANGE,
|
LANGUAGE_RANGE,
|
||||||
QUALITY_LIST,
|
QUALITY_LIST,
|
||||||
|
SAMPLE_RANGE,
|
||||||
SEASON_RANGE,
|
SEASON_RANGE,
|
||||||
SLOW_DELAY_RANGE,
|
SLOW_DELAY_RANGE,
|
||||||
SUBTITLE_CODEC,
|
SUBTITLE_CODEC,
|
||||||
@@ -118,7 +120,7 @@ from unshackle.core.utils.collections import ci_get, merge_dict
|
|||||||
from unshackle.core.utils.post_scripts import NO_POST_SCRIPTS, build_context, dispatch, season_context
|
from unshackle.core.utils.post_scripts import NO_POST_SCRIPTS, build_context, dispatch, season_context
|
||||||
from unshackle.core.utils.redact import mask_proxy
|
from unshackle.core.utils.redact import mask_proxy
|
||||||
from unshackle.core.utils.selector import select_multiple
|
from unshackle.core.utils.selector import select_multiple
|
||||||
from unshackle.core.utils.subprocess import ffmpeg_decodes, ffprobe
|
from unshackle.core.utils.subprocess import ffmpeg_decodes, ffprobe, run_step
|
||||||
from unshackle.core.vault import Vault
|
from unshackle.core.vault import Vault
|
||||||
from unshackle.core.vaults import Vaults
|
from unshackle.core.vaults import Vaults
|
||||||
|
|
||||||
@@ -259,6 +261,19 @@ def latest_episode_keys(titles: Any, latest_episode: bool, latest_episodes: Opti
|
|||||||
return list(dict.fromkeys((ep.season, ep.number) for ep in titles))[-count:]
|
return list(dict.fromkeys((ep.season, ep.number) for ep in titles))[-count:]
|
||||||
|
|
||||||
|
|
||||||
|
SAMPLE_DIR = "Sample"
|
||||||
|
|
||||||
|
|
||||||
|
def template_type(title: Any) -> str:
|
||||||
|
"""Return the output template key of a title: ``series``, ``songs`` or ``movies``."""
|
||||||
|
return "series" if isinstance(title, Episode) else "songs" if isinstance(title, Song) else "movies"
|
||||||
|
|
||||||
|
|
||||||
|
def sample_name(name: str, title: Any) -> str:
|
||||||
|
"""Return ``name`` with a SAMPLE tag, joined with the separator of the output template of the title."""
|
||||||
|
return f"{name.rstrip()}{config.get_template_separator(template_type(title))}SAMPLE"
|
||||||
|
|
||||||
|
|
||||||
def apply_enrich(title: Title_T, name: Optional[str], year: Optional[int], language: Optional[Language]) -> None:
|
def apply_enrich(title: Title_T, name: Optional[str], year: Optional[int], language: Optional[Language]) -> None:
|
||||||
"""Set the --enrich name, year and original language on a title. A song takes only the language.
|
"""Set the --enrich name, year and original language on a title. A song takes only the language.
|
||||||
|
|
||||||
@@ -324,6 +339,41 @@ def remux_music_tracks(title: Any, log: logging.Logger) -> None:
|
|||||||
log.warning(f"Could not remux audio track {track.id} for music: {e}")
|
log.warning(f"Could not remux audio track {track.id} for music: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
def cut_music_sample(title: Any, window: tuple[int, int], log: logging.Logger) -> None:
|
||||||
|
"""Cut each audio track of a Song to the sample window. A Song has no mux step that cuts it."""
|
||||||
|
if not isinstance(title, Song) or not binaries.FFMPEG:
|
||||||
|
return
|
||||||
|
for track in title.tracks.audio:
|
||||||
|
if not track.path or not track.path.exists():
|
||||||
|
continue
|
||||||
|
cut_path = track.path.with_stem(f"{track.path.stem}_cut")
|
||||||
|
try:
|
||||||
|
run_step(
|
||||||
|
[
|
||||||
|
binaries.FFMPEG,
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-y",
|
||||||
|
"-ss",
|
||||||
|
f"{max(window[0] - track.sample_start, 0):.3f}",
|
||||||
|
"-i",
|
||||||
|
track.path,
|
||||||
|
"-t",
|
||||||
|
str(window[1] - window[0]),
|
||||||
|
"-map",
|
||||||
|
"0",
|
||||||
|
"-c",
|
||||||
|
"copy",
|
||||||
|
cut_path,
|
||||||
|
],
|
||||||
|
output=cut_path,
|
||||||
|
label="music sample cut",
|
||||||
|
)
|
||||||
|
os.replace(cut_path, track.path)
|
||||||
|
except RuntimeError as e:
|
||||||
|
log.warning(f"Could not cut audio track {track.id} to the sample window, so it holds whole segments: {e}")
|
||||||
|
|
||||||
|
|
||||||
def apply_service_dl_overrides(ctx: click.Context, service_dl_config: dict[str, Any], log: logging.Logger) -> None:
|
def apply_service_dl_overrides(ctx: click.Context, service_dl_config: dict[str, Any], log: logging.Logger) -> None:
|
||||||
"""Apply ``services.<TAG>.dl`` config onto ``ctx.params``. Explicit CLI/env values win.
|
"""Apply ``services.<TAG>.dl`` config onto ``ctx.params``. Explicit CLI/env values win.
|
||||||
The config replaces defaults and global ``dl:`` default_map values."""
|
The config replaces defaults and global ``dl:`` default_map values."""
|
||||||
@@ -972,6 +1022,19 @@ class dl:
|
|||||||
"--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path."
|
"--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path."
|
||||||
)
|
)
|
||||||
@click.option("--no-mux", is_flag=True, default=False, help="Do not mux tracks into a container file.")
|
@click.option("--no-mux", is_flag=True, default=False, help="Do not mux tracks into a container file.")
|
||||||
|
@click.option(
|
||||||
|
"--sample",
|
||||||
|
type=SAMPLE_RANGE,
|
||||||
|
default=None,
|
||||||
|
help="Download a short sample and write it to a Sample folder. "
|
||||||
|
"Use --sample for 60 seconds from 2 minutes in, or --sample START-END (e.g., --sample 0-60) for a custom window.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--include-sample",
|
||||||
|
is_flag=True,
|
||||||
|
default=False,
|
||||||
|
help="Also cut a 60 second sample from each muxed file into a Sample folder next to it.",
|
||||||
|
)
|
||||||
@click.option(
|
@click.option(
|
||||||
"--postscript",
|
"--postscript",
|
||||||
"postscript",
|
"postscript",
|
||||||
@@ -1566,6 +1629,8 @@ class dl:
|
|||||||
no_folder: bool,
|
no_folder: bool,
|
||||||
no_source: bool,
|
no_source: bool,
|
||||||
no_mux: bool,
|
no_mux: bool,
|
||||||
|
sample: Optional[tuple[int, int]],
|
||||||
|
include_sample: bool,
|
||||||
workers: Optional[int],
|
workers: Optional[int],
|
||||||
adaptive_workers: bool,
|
adaptive_workers: bool,
|
||||||
download_processes: int,
|
download_processes: int,
|
||||||
@@ -1595,6 +1660,10 @@ class dl:
|
|||||||
postscript = NO_POST_SCRIPTS
|
postscript = NO_POST_SCRIPTS
|
||||||
if continue_downloads:
|
if continue_downloads:
|
||||||
config.continue_downloads = True
|
config.continue_downloads = True
|
||||||
|
config.sample = sample
|
||||||
|
if sample and include_sample:
|
||||||
|
raise click.ClickException("--sample and --include-sample cannot be used together.")
|
||||||
|
sample_dir = SAMPLE_DIR if sample else ""
|
||||||
self.tmdb_searched = False
|
self.tmdb_searched = False
|
||||||
self.search_source = None
|
self.search_source = None
|
||||||
self.service_anime = bool(getattr(service, "ANIME", False))
|
self.service_anime = bool(getattr(service, "ANIME", False))
|
||||||
@@ -3058,6 +3127,18 @@ class dl:
|
|||||||
self.prefer_vault_keys(title)
|
self.prefer_vault_keys(title)
|
||||||
self.cache_resolved_keys(title)
|
self.cache_resolved_keys(title)
|
||||||
|
|
||||||
|
if sample and not skip_dl:
|
||||||
|
whole_files = [
|
||||||
|
track
|
||||||
|
for track in (*title.tracks.videos, *title.tracks.audio)
|
||||||
|
if not track.can_sample(track.session or service.session)
|
||||||
|
]
|
||||||
|
if whole_files:
|
||||||
|
raise click.ClickException(
|
||||||
|
f"--sample cannot cut {len(whole_files)} track(s) of {title}: "
|
||||||
|
f"{self.service} gives each one as one file with no segment index."
|
||||||
|
)
|
||||||
|
|
||||||
dl_start_time = time.time()
|
dl_start_time = time.time()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -3213,6 +3294,10 @@ class dl:
|
|||||||
sub_conversions[(src, Subtitle.Codec.WebVTT.name)] = (
|
sub_conversions[(src, Subtitle.Codec.WebVTT.name)] = (
|
||||||
sub_conversions.get((src, Subtitle.Codec.WebVTT.name), 0) + 1
|
sub_conversions.get((src, Subtitle.Codec.WebVTT.name), 0) + 1
|
||||||
)
|
)
|
||||||
|
if sample:
|
||||||
|
for subtitle in title.tracks.subtitles:
|
||||||
|
if subtitle.path and subtitle.path.exists():
|
||||||
|
subtitle.trim(*sample)
|
||||||
for (src, dst), count in sub_conversions.items():
|
for (src, dst), count in sub_conversions.items():
|
||||||
log_event(
|
log_event(
|
||||||
"subtitle_convert",
|
"subtitle_convert",
|
||||||
@@ -3340,6 +3425,8 @@ class dl:
|
|||||||
self.log.warning(f"VUI normalization skipped for {track.id}: {e}")
|
self.log.warning(f"VUI normalization skipped for {track.id}: {e}")
|
||||||
|
|
||||||
remux_music_tracks(title, self.log)
|
remux_music_tracks(title, self.log)
|
||||||
|
if sample:
|
||||||
|
cut_music_sample(title, sample, self.log)
|
||||||
|
|
||||||
muxed_paths = []
|
muxed_paths = []
|
||||||
muxed_audio_codecs: dict[Path, Optional[Audio.Codec]] = {}
|
muxed_audio_codecs: dict[Path, Optional[Audio.Codec]] = {}
|
||||||
@@ -3522,6 +3609,11 @@ class dl:
|
|||||||
hybrid_output_path = config.directories.temp / hybrid_filename
|
hybrid_output_path = config.directories.temp / hybrid_filename
|
||||||
hybrid_temp_paths.append(hybrid_output_path)
|
hybrid_temp_paths.append(hybrid_output_path)
|
||||||
|
|
||||||
|
if len({track.sample_start for track in resolution_tracks}) > 1:
|
||||||
|
raise click.ClickException(
|
||||||
|
"--sample cut the HDR10 and Dolby Vision tracks at different "
|
||||||
|
"points, so a hybrid of them would not line up."
|
||||||
|
)
|
||||||
Hybrid(resolution_tracks, self.service, output_name=hybrid_filename)
|
Hybrid(resolution_tracks, self.service, output_name=hybrid_filename)
|
||||||
|
|
||||||
if container:
|
if container:
|
||||||
@@ -3579,6 +3671,8 @@ class dl:
|
|||||||
media_info = MediaInfo.parse(muxed_path) if muxed_path else None
|
media_info = MediaInfo.parse(muxed_path) if muxed_path else None
|
||||||
if media_info:
|
if media_info:
|
||||||
base_filename = title.get_filename(media_info, show_service=not no_source)
|
base_filename = title.get_filename(media_info, show_service=not no_source)
|
||||||
|
if sample:
|
||||||
|
base_filename = sample_name(base_filename, title)
|
||||||
else:
|
else:
|
||||||
base_filename = str(title)
|
base_filename = str(title)
|
||||||
|
|
||||||
@@ -3594,6 +3688,7 @@ class dl:
|
|||||||
sidecar_dir /= title.get_filename(
|
sidecar_dir /= title.get_filename(
|
||||||
media_info, show_service=not no_source, folder=True
|
media_info, show_service=not no_source, folder=True
|
||||||
)
|
)
|
||||||
|
sidecar_dir /= sample_dir
|
||||||
sidecar_targets.append((muxed_path, base_filename, sidecar_dir))
|
sidecar_targets.append((muxed_path, base_filename, sidecar_dir))
|
||||||
|
|
||||||
with console.status("Saving subtitle sidecar files..."):
|
with console.status("Saving subtitle sidecar files..."):
|
||||||
@@ -3674,11 +3769,14 @@ class dl:
|
|||||||
media_info = MediaInfo.parse(sample_track.path)
|
media_info = MediaInfo.parse(sample_track.path)
|
||||||
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
|
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
|
||||||
|
|
||||||
|
final_dir /= sample_dir
|
||||||
final_dir.mkdir(parents=True, exist_ok=True)
|
final_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
for track_path in muxed_paths:
|
for track_path in muxed_paths:
|
||||||
media_info = MediaInfo.parse(track_path)
|
media_info = MediaInfo.parse(track_path)
|
||||||
base_filename = title.get_filename(media_info, show_service=not no_source)
|
base_filename = title.get_filename(media_info, show_service=not no_source)
|
||||||
|
if sample:
|
||||||
|
base_filename = sample_name(base_filename, title)
|
||||||
|
|
||||||
track = next((t for t in title.tracks if t.path == track_path), None)
|
track = next((t for t in title.tracks if t.path == track_path), None)
|
||||||
if track:
|
if track:
|
||||||
@@ -3710,6 +3808,8 @@ class dl:
|
|||||||
media_info = MediaInfo.parse(muxed_path)
|
media_info = MediaInfo.parse(muxed_path)
|
||||||
final_dir = self.output_dir or config.directories.downloads
|
final_dir = self.output_dir or config.directories.downloads
|
||||||
final_filename = title.get_filename(media_info, show_service=not no_source)
|
final_filename = title.get_filename(media_info, show_service=not no_source)
|
||||||
|
if sample:
|
||||||
|
final_filename = sample_name(final_filename, title)
|
||||||
audio_codec_suffix = muxed_audio_codecs.get(muxed_path)
|
audio_codec_suffix = muxed_audio_codecs.get(muxed_path)
|
||||||
|
|
||||||
if not no_folder and (
|
if not no_folder and (
|
||||||
@@ -3717,13 +3817,11 @@ class dl:
|
|||||||
or (isinstance(title, Movie) and config.get_folder_template("movies"))
|
or (isinstance(title, Movie) and config.get_folder_template("movies"))
|
||||||
):
|
):
|
||||||
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
|
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
|
||||||
|
final_dir /= sample_dir
|
||||||
|
|
||||||
final_dir.mkdir(parents=True, exist_ok=True)
|
final_dir.mkdir(parents=True, exist_ok=True)
|
||||||
final_path = final_dir / f"{final_filename}{muxed_path.suffix}"
|
final_path = final_dir / f"{final_filename}{muxed_path.suffix}"
|
||||||
template_type = (
|
sep = config.get_template_separator(template_type(title))
|
||||||
"series" if isinstance(title, Episode) else "songs" if isinstance(title, Song) else "movies"
|
|
||||||
)
|
|
||||||
sep = config.get_template_separator(template_type)
|
|
||||||
|
|
||||||
if final_path.exists() and audio_codec_suffix and append_audio_codec_suffix:
|
if final_path.exists() and audio_codec_suffix and append_audio_codec_suffix:
|
||||||
final_filename = f"{final_filename.rstrip()}{sep}{audio_codec_suffix.name}"
|
final_filename = f"{final_filename.rstrip()}{sep}{audio_codec_suffix.name}"
|
||||||
@@ -3754,6 +3852,8 @@ class dl:
|
|||||||
self.anime_hint(title),
|
self.anime_hint(title),
|
||||||
session=service.session,
|
session=service.session,
|
||||||
)
|
)
|
||||||
|
if include_sample:
|
||||||
|
self.write_sample(final_path, title, media_info)
|
||||||
|
|
||||||
post_script_context = build_context(
|
post_script_context = build_context(
|
||||||
title,
|
title,
|
||||||
@@ -3807,6 +3907,32 @@ class dl:
|
|||||||
|
|
||||||
console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5)))
|
console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5)))
|
||||||
|
|
||||||
|
def write_sample(self, source: Path, title: Title_T, media_info: MediaInfo) -> None:
|
||||||
|
"""Cut a 60 second sample from a finished file into a ``Sample`` folder next to it.
|
||||||
|
|
||||||
|
The sample starts 2 minutes in. A file shorter than 3 minutes gives its middle 60 seconds.
|
||||||
|
"""
|
||||||
|
duration = float(media_info.general_tracks[0].duration or 0) / 1000
|
||||||
|
start, end = DEFAULT_SAMPLE
|
||||||
|
length = end - start
|
||||||
|
if duration < end:
|
||||||
|
start = max(0.0, (duration - length) / 2)
|
||||||
|
output = source.parent / SAMPLE_DIR / f"{sample_name(source.stem, title)}{source.suffix}"
|
||||||
|
output.parent.mkdir(exist_ok=True)
|
||||||
|
returncode = subprocess.run(
|
||||||
|
[
|
||||||
|
str(binaries.MKVToolNix),
|
||||||
|
"-q",
|
||||||
|
"-o",
|
||||||
|
str(output),
|
||||||
|
"--split",
|
||||||
|
f"parts:{start:.3f}s-{start + length:.3f}s",
|
||||||
|
source,
|
||||||
|
]
|
||||||
|
).returncode
|
||||||
|
if returncode > 1 or not output.exists():
|
||||||
|
self.log.warning(f"mkvmerge wrote no sample for {source.name}")
|
||||||
|
|
||||||
def anime_hint(self, title: Optional[Title_T] = None) -> bool:
|
def anime_hint(self, title: Optional[Title_T] = None) -> bool:
|
||||||
"""Whether metadata lookups for this title should prefer AniList."""
|
"""Whether metadata lookups for this title should prefer AniList."""
|
||||||
per_title = getattr(title, "anime", None)
|
per_title = getattr(title, "anime", None)
|
||||||
|
|||||||
@@ -418,6 +418,15 @@ def perform_download(
|
|||||||
if params.get("export"):
|
if params.get("export"):
|
||||||
params["export"] = bool(params["export"])
|
params["export"] = bool(params["export"])
|
||||||
|
|
||||||
|
sample_raw = params.get("sample")
|
||||||
|
if sample_raw is not None:
|
||||||
|
from unshackle.core.utils.click_types import SAMPLE_RANGE
|
||||||
|
|
||||||
|
try:
|
||||||
|
params["sample"] = SAMPLE_RANGE.convert(sample_raw, None, None)
|
||||||
|
except click.BadParameter as exc:
|
||||||
|
raise Exception(f"Invalid sample parameter: {exc}")
|
||||||
|
|
||||||
slow_raw = params.get("slow")
|
slow_raw = params.get("slow")
|
||||||
if slow_raw is not None and not isinstance(slow_raw, tuple):
|
if slow_raw is not None and not isinstance(slow_raw, tuple):
|
||||||
if isinstance(slow_raw, bool):
|
if isinstance(slow_raw, bool):
|
||||||
@@ -633,6 +642,8 @@ def perform_download(
|
|||||||
no_folder=params.get("no_folder", False),
|
no_folder=params.get("no_folder", False),
|
||||||
no_source=params.get("no_source", False),
|
no_source=params.get("no_source", False),
|
||||||
no_mux=params.get("no_mux", False),
|
no_mux=params.get("no_mux", False),
|
||||||
|
sample=params.get("sample"),
|
||||||
|
include_sample=params.get("include_sample", False),
|
||||||
workers=params.get("workers"),
|
workers=params.get("workers"),
|
||||||
adaptive_workers=params.get("adaptive_workers", False),
|
adaptive_workers=params.get("adaptive_workers", False),
|
||||||
download_processes=params.get("download_processes", 1),
|
download_processes=params.get("download_processes", 1),
|
||||||
|
|||||||
@@ -92,6 +92,8 @@ DEFAULT_DOWNLOAD_PARAMS = {
|
|||||||
"no_folder": False,
|
"no_folder": False,
|
||||||
"no_source": False,
|
"no_source": False,
|
||||||
"no_mux": False,
|
"no_mux": False,
|
||||||
|
"sample": None,
|
||||||
|
"include_sample": False,
|
||||||
"workers": None,
|
"workers": None,
|
||||||
"adaptive_workers": False,
|
"adaptive_workers": False,
|
||||||
"download_processes": 1,
|
"download_processes": 1,
|
||||||
@@ -1831,6 +1833,16 @@ def validate_download_parameters(data: Dict[str, Any]) -> Optional[str]:
|
|||||||
except APIError as e:
|
except APIError as e:
|
||||||
return e.message
|
return e.message
|
||||||
|
|
||||||
|
if data.get("sample") is not None:
|
||||||
|
from unshackle.core.utils.click_types import SAMPLE_RANGE
|
||||||
|
|
||||||
|
try:
|
||||||
|
SAMPLE_RANGE.convert(data["sample"], None, None)
|
||||||
|
except click.BadParameter:
|
||||||
|
return "sample must be true or a 'START-END' window in seconds, such as '60-120'."
|
||||||
|
if data.get("sample") and data.get("include_sample"):
|
||||||
|
return "sample and include_sample cannot be used together."
|
||||||
|
|
||||||
# The serve user would create and write any directory a client names; keep it under downloads.
|
# The serve user would create and write any directory a client names; keep it under downloads.
|
||||||
if data.get("output_dir"):
|
if data.get("output_dir"):
|
||||||
root = config.directories.downloads.resolve()
|
root = config.directories.downloads.resolve()
|
||||||
|
|||||||
@@ -1005,6 +1005,14 @@ async def download(request: web.Request) -> web.Response:
|
|||||||
no_mux:
|
no_mux:
|
||||||
type: boolean
|
type: boolean
|
||||||
description: Do not mux tracks into a container file (default - false)
|
description: Do not mux tracks into a container file (default - false)
|
||||||
|
include_sample:
|
||||||
|
type: boolean
|
||||||
|
description: Also cut a 60 second sample from each muxed file into a Sample folder next to it (default - false)
|
||||||
|
sample:
|
||||||
|
oneOf:
|
||||||
|
- type: boolean
|
||||||
|
- type: string
|
||||||
|
description: Download a short sample and write it to a Sample folder. `true` for 60 seconds from 2 minutes in, or a `"START-END"` string in seconds (e.g., `"0-60"`) (default - null)
|
||||||
workers:
|
workers:
|
||||||
type: integer
|
type: integer
|
||||||
description: Max workers/threads per track download (default - None)
|
description: Max workers/threads per track download (default - None)
|
||||||
|
|||||||
@@ -193,6 +193,7 @@ class Config:
|
|||||||
# mask local base dirs (install root/venv/home) in logged paths; False shows full paths
|
# mask local base dirs (install root/venv/home) in logged paths; False shows full paths
|
||||||
self.redact_paths: bool = kwargs.get("redact_paths", True)
|
self.redact_paths: bool = kwargs.get("redact_paths", True)
|
||||||
self.continue_downloads: bool = kwargs.get("continue_downloads", False)
|
self.continue_downloads: bool = kwargs.get("continue_downloads", False)
|
||||||
|
self.sample: Optional[tuple[int, int]] = None
|
||||||
self.post_scripts: list = kwargs.get("post_scripts") or []
|
self.post_scripts: list = kwargs.get("post_scripts") or []
|
||||||
|
|
||||||
self.language_tags: dict = kwargs.get("language_tags") or {}
|
self.language_tags: dict = kwargs.get("language_tags") or {}
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ from typing import TypeVar, Union
|
|||||||
|
|
||||||
DOWNLOAD_CANCELLED = Event()
|
DOWNLOAD_CANCELLED = Event()
|
||||||
DOWNLOAD_LICENCE_ONLY = Event()
|
DOWNLOAD_LICENCE_ONLY = Event()
|
||||||
|
|
||||||
|
DEFAULT_SAMPLE = (120, 180)
|
||||||
DOWNLOAD_ALL_DRM = Event()
|
DOWNLOAD_ALL_DRM = Event()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -34,7 +34,11 @@ from unshackle.core.drm.verify import decrypt_track
|
|||||||
from unshackle.core.events import events
|
from unshackle.core.events import events
|
||||||
from unshackle.core.session import RnetSession
|
from unshackle.core.session import RnetSession
|
||||||
from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Tracks, Video, resume
|
from unshackle.core.tracks import Audio, DownloadContext, Subtitle, Tracks, Video, resume
|
||||||
from unshackle.core.tracks.track import DRM_PREFERENCE_TYPES, assert_fragments_decrypted
|
from unshackle.core.tracks.track import (
|
||||||
|
DRM_PREFERENCE_TYPES,
|
||||||
|
assert_fragments_decrypted,
|
||||||
|
parse_sidx,
|
||||||
|
)
|
||||||
from unshackle.core.utilities import is_close_match, log_event, try_ensure_utf8
|
from unshackle.core.utilities import is_close_match, log_event, try_ensure_utf8
|
||||||
from unshackle.core.utils.redact import safe_display_url
|
from unshackle.core.utils.redact import safe_display_url
|
||||||
from unshackle.core.utils.xml import load_xml
|
from unshackle.core.utils.xml import load_xml
|
||||||
@@ -458,6 +462,7 @@ class DASH:
|
|||||||
track_url=track.url,
|
track_url=track.url,
|
||||||
session=session,
|
session=session,
|
||||||
probe_kid=period_idx == 0,
|
probe_kid=period_idx == 0,
|
||||||
|
window=config.sample,
|
||||||
)
|
)
|
||||||
|
|
||||||
if period_idx == 0:
|
if period_idx == 0:
|
||||||
@@ -471,12 +476,20 @@ class DASH:
|
|||||||
seen_segments.add(seg)
|
seen_segments.add(seg)
|
||||||
segments.append(seg)
|
segments.append(seg)
|
||||||
segment_durations.extend(p_durations)
|
segment_durations.extend(p_durations)
|
||||||
|
# ponytail: a sample comes from the first content period only; map the window across
|
||||||
|
# periods if a manifest with a short first period needs it
|
||||||
|
if config.sample and segments:
|
||||||
|
break
|
||||||
|
|
||||||
if not segments:
|
if not segments:
|
||||||
log.error("Could not find a way to get segments from this MPD manifest.")
|
log.error("Could not find a way to get segments from this MPD manifest.")
|
||||||
log.debug(track.url)
|
log.debug(track.url)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
if config.sample and not segment_durations and not isinstance(track, Subtitle):
|
||||||
|
log.error("--sample cannot cut this track: the manifest gives it as one file with no segment index.")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
# TODO: Should we floor/ceil/round, or is int() ok?
|
# TODO: Should we floor/ceil/round, or is int() ok?
|
||||||
track.data["dash"]["timescale"] = int(segment_timescale)
|
track.data["dash"]["timescale"] = int(segment_timescale)
|
||||||
track.data["dash"]["segment_durations"] = segment_durations
|
track.data["dash"]["segment_durations"] = segment_durations
|
||||||
@@ -533,7 +546,7 @@ class DASH:
|
|||||||
# When every segment is a byte range of one parent resource and init is its [0, len)
|
# When every segment is a byte range of one parent resource and init is its [0, len)
|
||||||
# prefix, the whole resource is itself a valid MP4, so one direct download to save_path
|
# prefix, the whole resource is itself a valid MP4, so one direct download to save_path
|
||||||
# replaces the per-segment files, init write and merge pass.
|
# replaces the per-segment files, init write and merge pass.
|
||||||
collapse_single_url = DASH.collapsible_single_url(
|
collapse_single_url = not config.sample and DASH.collapsible_single_url(
|
||||||
isinstance(track, Subtitle),
|
isinstance(track, Subtitle),
|
||||||
segments,
|
segments,
|
||||||
len(init_data) if init_data is not None else None,
|
len(init_data) if init_data is not None else None,
|
||||||
@@ -753,6 +766,26 @@ class DASH:
|
|||||||
|
|
||||||
progress(downloaded="Downloaded")
|
progress(downloaded="Downloaded")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def find_segment_element(name: str, adaptation_set: Element, representation: Element) -> Optional[Element]:
|
||||||
|
"""Find a segment description element on the representation, or else on its adaptation set."""
|
||||||
|
element = representation.find(name)
|
||||||
|
return element if element is not None else adaptation_set.find(name)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def has_segment_timing(track: AnyTrack) -> bool:
|
||||||
|
"""Return False when the manifest gives the track as one file with no index, so a sample cannot cut it."""
|
||||||
|
dash = track.data.get("dash") or {}
|
||||||
|
representation, adaptation_set = dash.get("representation"), dash.get("adaptation_set")
|
||||||
|
if representation is None or adaptation_set is None:
|
||||||
|
return True
|
||||||
|
segment_base = DASH.find_segment_element("SegmentBase", adaptation_set, representation)
|
||||||
|
return (
|
||||||
|
DASH.merge_segment_templates(adaptation_set, representation) is not None
|
||||||
|
or DASH.find_segment_element("SegmentList", adaptation_set, representation) is not None
|
||||||
|
or (segment_base is not None and bool(segment_base.get("indexRange")))
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def collapsible_single_url(
|
def collapsible_single_url(
|
||||||
is_subtitle: bool,
|
is_subtitle: bool,
|
||||||
@@ -844,6 +877,7 @@ class DASH:
|
|||||||
track_url: str,
|
track_url: str,
|
||||||
session: Union[Session, RnetSession],
|
session: Union[Session, RnetSession],
|
||||||
probe_kid: bool = True,
|
probe_kid: bool = True,
|
||||||
|
window: Optional[tuple[int, int]] = None,
|
||||||
) -> tuple[
|
) -> tuple[
|
||||||
Optional[bytes],
|
Optional[bytes],
|
||||||
list[tuple[str, Optional[str]]],
|
list[tuple[str, Optional[str]]],
|
||||||
@@ -877,13 +911,9 @@ class DASH:
|
|||||||
|
|
||||||
segment_template = DASH.merge_segment_templates(adaptation_set, representation)
|
segment_template = DASH.merge_segment_templates(adaptation_set, representation)
|
||||||
|
|
||||||
segment_list = representation.find("SegmentList")
|
segment_list = DASH.find_segment_element("SegmentList", adaptation_set, representation)
|
||||||
if segment_list is None:
|
|
||||||
segment_list = adaptation_set.find("SegmentList")
|
|
||||||
|
|
||||||
segment_base = representation.find("SegmentBase")
|
segment_base = DASH.find_segment_element("SegmentBase", adaptation_set, representation)
|
||||||
if segment_base is None:
|
|
||||||
segment_base = adaptation_set.find("SegmentBase")
|
|
||||||
|
|
||||||
segments: list[tuple[str, Optional[str]]] = []
|
segments: list[tuple[str, Optional[str]]] = []
|
||||||
segment_timescale: float = 0
|
segment_timescale: float = 0
|
||||||
@@ -930,6 +960,10 @@ class DASH:
|
|||||||
segment_durations.append(current_time)
|
segment_durations.append(current_time)
|
||||||
current_time += int(s.get("d"))
|
current_time += int(s.get("d"))
|
||||||
|
|
||||||
|
keep = track.sample_slice(segment_durations, segment_timescale, window) if window else slice(None)
|
||||||
|
start_number += keep.start or 0
|
||||||
|
segment_durations = segment_durations[keep]
|
||||||
|
|
||||||
if not end_number:
|
if not end_number:
|
||||||
end_number = start_number + len(segment_durations) - 1
|
end_number = start_number + len(segment_durations) - 1
|
||||||
|
|
||||||
@@ -956,7 +990,11 @@ class DASH:
|
|||||||
segment_count = math.ceil(period_duration / (segment_duration / segment_timescale))
|
segment_count = math.ceil(period_duration / (segment_duration / segment_timescale))
|
||||||
end_number = start_number + segment_count - 1
|
end_number = start_number + segment_count - 1
|
||||||
|
|
||||||
for s in range(start_number, end_number + 1):
|
numbers = range(start_number, end_number + 1)
|
||||||
|
if window:
|
||||||
|
starts = [n * segment_duration for n in range(len(numbers))]
|
||||||
|
numbers = numbers[track.sample_slice(starts, segment_timescale, window)]
|
||||||
|
for s in numbers:
|
||||||
segments.append(
|
segments.append(
|
||||||
(
|
(
|
||||||
DASH.replace_fields(
|
DASH.replace_fields(
|
||||||
@@ -994,6 +1032,10 @@ class DASH:
|
|||||||
track_kid = track.get_key_id(init_data)
|
track_kid = track.get_key_id(init_data)
|
||||||
|
|
||||||
segment_urls = segment_list.findall("SegmentURL")
|
segment_urls = segment_list.findall("SegmentURL")
|
||||||
|
list_duration = float(segment_list.get("duration") or 0)
|
||||||
|
if window and list_duration:
|
||||||
|
starts = [n * list_duration for n in range(len(segment_urls))]
|
||||||
|
segment_urls = segment_urls[track.sample_slice(starts, segment_timescale, window)]
|
||||||
for segment_url in segment_urls:
|
for segment_url in segment_urls:
|
||||||
media_url = segment_url.get("media")
|
media_url = segment_url.get("media")
|
||||||
if not media_url:
|
if not media_url:
|
||||||
@@ -1022,7 +1064,20 @@ class DASH:
|
|||||||
if total_size:
|
if total_size:
|
||||||
media_range = f"{len(init_data)}-{total_size}"
|
media_range = f"{len(init_data)}-{total_size}"
|
||||||
|
|
||||||
segments.append((rep_base_url, media_range))
|
subsegments: list[tuple[int, int, int]] = []
|
||||||
|
index_range = segment_base.get("indexRange")
|
||||||
|
if window and index_range and not isinstance(track, Subtitle):
|
||||||
|
res = session.get(url=rep_base_url, headers={"Range": f"bytes={index_range}"})
|
||||||
|
res.raise_for_status()
|
||||||
|
segment_timescale, subsegments = parse_sidx(res.content, int(index_range.split("-")[0]))
|
||||||
|
|
||||||
|
if subsegments:
|
||||||
|
segment_durations = [time for time, _, _ in subsegments]
|
||||||
|
keep = track.sample_slice(segment_durations, segment_timescale, window)
|
||||||
|
segment_durations = segment_durations[keep]
|
||||||
|
segments.extend((rep_base_url, f"{first}-{last}") for _, first, last in subsegments[keep])
|
||||||
|
else:
|
||||||
|
segments.append((rep_base_url, media_range))
|
||||||
elif rep_base_url:
|
elif rep_base_url:
|
||||||
segments.append((rep_base_url, None))
|
segments.append((rep_base_url, None))
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import subprocess
|
|||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from contextlib import ExitStack, closing
|
from contextlib import ExitStack, closing
|
||||||
|
from itertools import accumulate
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Optional, Union
|
from typing import Any, Optional, Union
|
||||||
from urllib.parse import urljoin, urlparse
|
from urllib.parse import urljoin, urlparse
|
||||||
@@ -685,6 +686,12 @@ class HLS:
|
|||||||
)
|
)
|
||||||
hls_data["unwanted_segments"] = [s.absolute_uri for s in unwanted_segments]
|
hls_data["unwanted_segments"] = [s.absolute_uri for s in unwanted_segments]
|
||||||
|
|
||||||
|
if config.sample:
|
||||||
|
content = [segment for segment in master.segments if segment not in unwanted_segments]
|
||||||
|
starts = list(accumulate((segment.duration or 0 for segment in content[:-1]), initial=0.0))
|
||||||
|
del content[track.sample_slice(starts, window=config.sample)]
|
||||||
|
unwanted_segments += content
|
||||||
|
|
||||||
# Get session DRM as fallback but prefer media playlist keys for accurate KID matching
|
# Get session DRM as fallback but prefer media playlist keys for accurate KID matching
|
||||||
if track.drm:
|
if track.drm:
|
||||||
session_drm = track.get_drm_for_cdm(cdm)
|
session_drm = track.get_drm_for_cdm(cdm)
|
||||||
@@ -796,17 +803,17 @@ class HLS:
|
|||||||
|
|
||||||
range_offset = 0
|
range_offset = 0
|
||||||
for segment in master.segments:
|
for segment in master.segments:
|
||||||
|
if segment.byterange:
|
||||||
|
byte_range = HLS.calculate_byte_range(segment.byterange, range_offset)
|
||||||
|
range_offset = int(byte_range.split("-")[1]) + 1
|
||||||
|
else:
|
||||||
|
byte_range = None
|
||||||
|
|
||||||
if segment in unwanted_segments:
|
if segment in unwanted_segments:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
segment_durations.append(int(segment.duration))
|
segment_durations.append(int(segment.duration))
|
||||||
|
|
||||||
if segment.byterange:
|
|
||||||
byte_range = HLS.calculate_byte_range(segment.byterange, range_offset)
|
|
||||||
range_offset = int(byte_range.split("-")[0])
|
|
||||||
else:
|
|
||||||
byte_range = None
|
|
||||||
|
|
||||||
urls.append(
|
urls.append(
|
||||||
{
|
{
|
||||||
"url": urljoin(segment.base_uri, segment.uri),
|
"url": urljoin(segment.base_uri, segment.uri),
|
||||||
|
|||||||
@@ -259,6 +259,7 @@ class ISM:
|
|||||||
)
|
)
|
||||||
|
|
||||||
track_urls: list[str] = []
|
track_urls: list[str] = []
|
||||||
|
track_times: list[int] = []
|
||||||
fragment_time = 0
|
fragment_time = 0
|
||||||
fragments = stream_index.findall("c")
|
fragments = stream_index.findall("c")
|
||||||
# MS-SSTR UrlPattern; regex over str.format so {Bitrate}/{start_time}
|
# MS-SSTR UrlPattern; regex over str.format so {Bitrate}/{start_time}
|
||||||
@@ -276,6 +277,7 @@ class ISM:
|
|||||||
first_time = int(fragments[0].get("t") or 0)
|
first_time = int(fragments[0].get("t") or 0)
|
||||||
if first_time != 0:
|
if first_time != 0:
|
||||||
track_urls.append(re.sub(r"\{start[ _]time\}", "0", url_template))
|
track_urls.append(re.sub(r"\{start[ _]time\}", "0", url_template))
|
||||||
|
track_times.append(first_time)
|
||||||
|
|
||||||
for idx, frag in enumerate(fragments):
|
for idx, frag in enumerate(fragments):
|
||||||
fragment_time = int(frag.get("t", fragment_time))
|
fragment_time = int(frag.get("t", fragment_time))
|
||||||
@@ -292,6 +294,7 @@ class ISM:
|
|||||||
duration_frag = (next_time - fragment_time) // repeat
|
duration_frag = (next_time - fragment_time) // repeat
|
||||||
for _ in range(repeat):
|
for _ in range(repeat):
|
||||||
track_urls.append(re.sub(r"\{start[ _]time\}", str(fragment_time), url_template))
|
track_urls.append(re.sub(r"\{start[ _]time\}", str(fragment_time), url_template))
|
||||||
|
track_times.append(fragment_time)
|
||||||
fragment_time += duration_frag
|
fragment_time += duration_frag
|
||||||
|
|
||||||
track_id = hashlib.md5(
|
track_id = hashlib.md5(
|
||||||
@@ -312,6 +315,8 @@ class ISM:
|
|||||||
"stream_index": stream_index,
|
"stream_index": stream_index,
|
||||||
"quality_level": ql,
|
"quality_level": ql,
|
||||||
"segments": track_urls,
|
"segments": track_urls,
|
||||||
|
"segment_times": track_times,
|
||||||
|
"timescale": int(stream_index.get("TimeScale") or self.manifest.get("TimeScale") or 10000000),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -414,6 +419,9 @@ class ISM:
|
|||||||
session.proxies.update({"all": proxy})
|
session.proxies.update({"all": proxy})
|
||||||
|
|
||||||
segments: list[str] = track.data["ism"]["segments"]
|
segments: list[str] = track.data["ism"]["segments"]
|
||||||
|
segment_times: list[int] = track.data["ism"].get("segment_times") or []
|
||||||
|
if segment_times:
|
||||||
|
segments = segments[track.sample_slice(segment_times, track.data["ism"]["timescale"], config.sample)]
|
||||||
|
|
||||||
session_drm = None
|
session_drm = None
|
||||||
if track.drm:
|
if track.drm:
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import click
|
|||||||
|
|
||||||
from unshackle.core import binaries
|
from unshackle.core import binaries
|
||||||
from unshackle.core.config import config
|
from unshackle.core.config import config
|
||||||
|
from unshackle.core.constants import DEFAULT_SAMPLE
|
||||||
from unshackle.core.service import Service
|
from unshackle.core.service import Service
|
||||||
from unshackle.core.service_repo import DirtyServiceRepo, head, is_repo_spec, refresh_repo, resolve_service_repo
|
from unshackle.core.service_repo import DirtyServiceRepo, head, is_repo_spec, refresh_repo, resolve_service_repo
|
||||||
from unshackle.core.utilities import import_module_by_path
|
from unshackle.core.utilities import import_module_by_path
|
||||||
@@ -360,16 +361,17 @@ class Services(click.Group):
|
|||||||
super().format_help(ctx, formatter)
|
super().format_help(ctx, formatter)
|
||||||
|
|
||||||
def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:
|
def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:
|
||||||
"""Preprocess ``--slow`` so it can take an optional range value before Click parses the arguments."""
|
"""Preprocess ``--slow`` and ``--sample`` so each can take an optional range value before Click parses."""
|
||||||
|
defaults = {"--slow": "60-120", "--sample": "{}-{}".format(*DEFAULT_SAMPLE)}
|
||||||
processed = []
|
processed = []
|
||||||
i = 0
|
i = 0
|
||||||
while i < len(args):
|
while i < len(args):
|
||||||
if args[i] == "--slow":
|
if args[i] in defaults:
|
||||||
if i + 1 < len(args) and re.match(r"^\d+-\d+$", args[i + 1]):
|
if i + 1 < len(args) and re.match(r"^\d+-\d+$", args[i + 1]):
|
||||||
processed.append(f"--slow={args[i + 1]}")
|
processed.append(f"{args[i]}={args[i + 1]}")
|
||||||
i += 2
|
i += 2
|
||||||
else:
|
else:
|
||||||
processed.append("--slow=60-120")
|
processed.append(f"{args[i]}={defaults[args[i]]}")
|
||||||
i += 1
|
i += 1
|
||||||
else:
|
else:
|
||||||
processed.append(args[i])
|
processed.append(args[i])
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from enum import Enum
|
|||||||
from functools import partial
|
from functools import partial
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Callable, Iterable, Optional, Union
|
from typing import Any, Callable, Iterable, Optional, Sequence, Union
|
||||||
|
|
||||||
import pycaption
|
import pycaption
|
||||||
import requests
|
import requests
|
||||||
@@ -1001,6 +1001,32 @@ class Subtitle(Track):
|
|||||||
|
|
||||||
return captions, language
|
return captions, language
|
||||||
|
|
||||||
|
def sample_slice(
|
||||||
|
self, starts: Sequence[float], timescale: float = 1, window: Optional[tuple[int, int]] = None
|
||||||
|
) -> slice:
|
||||||
|
"""Keep every segment: ``trim`` cuts a subtitle by its cue times after the download."""
|
||||||
|
return slice(None)
|
||||||
|
|
||||||
|
def trim(self, start: float, end: float) -> bool:
|
||||||
|
"""Keep the cues between ``start`` and ``end`` seconds and move them to start at zero.
|
||||||
|
|
||||||
|
Returns False and leaves the file whole when pysubs2 cannot read or write its format.
|
||||||
|
"""
|
||||||
|
import pysubs2
|
||||||
|
|
||||||
|
low, high = round(start * 1000), round(end * 1000)
|
||||||
|
try:
|
||||||
|
subs = pysubs2.load(str(self.path), encoding="utf-8")
|
||||||
|
subs.events = [event for event in subs.events if event.end > low and event.start < high]
|
||||||
|
for event in subs.events:
|
||||||
|
event.start = max(event.start, low) - low
|
||||||
|
event.end = min(event.end, high) - low
|
||||||
|
subs.save(str(self.path))
|
||||||
|
except (pysubs2.exceptions.Pysubs2Error, UnicodeDecodeError):
|
||||||
|
return False
|
||||||
|
self.sample_start = start
|
||||||
|
return True
|
||||||
|
|
||||||
def strip_hearing_impaired(self) -> None:
|
def strip_hearing_impaired(self) -> None:
|
||||||
"""
|
"""
|
||||||
Remove the SDH captions.
|
Remove the SDH captions.
|
||||||
|
|||||||
@@ -3,18 +3,23 @@ import html
|
|||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
|
import struct
|
||||||
import subprocess
|
import subprocess
|
||||||
|
from bisect import bisect_left, bisect_right
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from copy import copy
|
from copy import copy
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Callable, Iterable, Optional, Union
|
from typing import Any, Callable, Iterable, Optional, Sequence, Union
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
from zlib import crc32
|
from zlib import crc32
|
||||||
|
|
||||||
|
import click
|
||||||
|
from construct import ConstructError
|
||||||
from langcodes import Language
|
from langcodes import Language
|
||||||
|
from pymp4.parser import Box
|
||||||
from requests import Response, Session
|
from requests import Response, Session
|
||||||
from requests.adapters import HTTPAdapter, Retry
|
from requests.adapters import HTTPAdapter, Retry
|
||||||
|
|
||||||
@@ -364,6 +369,31 @@ def assert_fragments_decrypted(path: Path) -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
SAMPLE_PROBE_BYTES = 64 * 1024
|
||||||
|
SAMPLE_PROBE_LIMIT = 8 * 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
def parse_sidx(data: bytes, offset: int) -> tuple[int, list[tuple[int, int, int]]]:
|
||||||
|
"""Read a ``sidx`` box that starts at byte ``offset`` of its file.
|
||||||
|
|
||||||
|
Returns the timescale and one (start time, first byte, last byte) for each subsegment. The list
|
||||||
|
is empty when the box is not a ``sidx`` or when it points at more indexes and not at media.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
box = Box.parse(data)
|
||||||
|
except ConstructError:
|
||||||
|
return 0, []
|
||||||
|
if box.type != b"sidx" or any(reference.reference_type != "MEDIA" for reference in box.references):
|
||||||
|
return 0, []
|
||||||
|
start, byte = box.earliest_presentation_time, offset + box.end + box.first_offset
|
||||||
|
subsegments: list[tuple[int, int, int]] = []
|
||||||
|
for reference in box.references:
|
||||||
|
subsegments.append((start, byte, byte + reference.referenced_size - 1))
|
||||||
|
start += reference.segment_duration
|
||||||
|
byte += reference.referenced_size
|
||||||
|
return box.timescale, subsegments
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class DownloadContext:
|
class DownloadContext:
|
||||||
"""Shared arguments passed to each manifest's ``download_track``."""
|
"""Shared arguments passed to each manifest's ``download_track``."""
|
||||||
@@ -396,6 +426,8 @@ class Track:
|
|||||||
DASH = 3 # https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
|
DASH = 3 # https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
|
||||||
ISM = 4 # https://learn.microsoft.com/en-us/silverlight/smooth-streaming
|
ISM = 4 # https://learn.microsoft.com/en-us/silverlight/smooth-streaming
|
||||||
|
|
||||||
|
sample_tail = 0
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
url: Union[str, list[str]],
|
url: Union[str, list[str]],
|
||||||
@@ -497,6 +529,77 @@ class Track:
|
|||||||
|
|
||||||
# TODO: Currently using OnFoo event naming, change to just segment_filter
|
# TODO: Currently using OnFoo event naming, change to just segment_filter
|
||||||
self.OnSegmentFilter: Optional[Callable] = None
|
self.OnSegmentFilter: Optional[Callable] = None
|
||||||
|
self.sample_start: float = 0.0
|
||||||
|
self.sample_bytes: Optional[tuple[bytes, str]] = None
|
||||||
|
|
||||||
|
def sample_slice(
|
||||||
|
self, starts: Sequence[float], timescale: float = 1, window: Optional[tuple[int, int]] = None
|
||||||
|
) -> slice:
|
||||||
|
"""Return the slice of segments that hold ``window``, from the start time of each segment.
|
||||||
|
|
||||||
|
Sets ``sample_start`` to the start time of the first kept segment, in seconds. A video keeps
|
||||||
|
``sample_tail`` segments after the window: mkvmerge ends the cut only at a video keyframe, and
|
||||||
|
with no such keyframe the audio and the subtitles continue to their own end.
|
||||||
|
"""
|
||||||
|
self.sample_start = 0.0
|
||||||
|
if not window or not starts:
|
||||||
|
return slice(None)
|
||||||
|
low, high = (starts[0] + seconds * timescale for seconds in window)
|
||||||
|
if low > starts[-1] and len(starts) > 1:
|
||||||
|
raise click.ClickException(
|
||||||
|
"The title ends before the sample window starts. Use --sample START-END with an earlier window."
|
||||||
|
)
|
||||||
|
first = max(bisect_right(starts, low) - 1, 0)
|
||||||
|
self.sample_start = (starts[first] - starts[0]) / timescale
|
||||||
|
return slice(first, max(bisect_left(starts, high) + self.sample_tail, first + 1))
|
||||||
|
|
||||||
|
def sample_byte_range(self, session: Union[Session, "RnetSession"]) -> Optional[tuple[bytes, str]]:
|
||||||
|
"""Find the bytes of a one-file track that hold the sample window.
|
||||||
|
|
||||||
|
Reads the start of the file for a ``sidx`` box. Returns the bytes before that box, which hold
|
||||||
|
the init data, and the byte range of the window. Returns None when the file has no such index.
|
||||||
|
"""
|
||||||
|
if self.sample_bytes or not isinstance(self.url, str):
|
||||||
|
return self.sample_bytes
|
||||||
|
head = b""
|
||||||
|
|
||||||
|
def read_to(end: int) -> None:
|
||||||
|
nonlocal head
|
||||||
|
if end > len(head):
|
||||||
|
res = session.get(self.url, headers={"Range": f"bytes={len(head)}-{end - 1}"})
|
||||||
|
if res.ok:
|
||||||
|
head += res.content
|
||||||
|
|
||||||
|
pos = 0
|
||||||
|
while pos < SAMPLE_PROBE_LIMIT:
|
||||||
|
if pos + 8 > len(head):
|
||||||
|
read_to(pos + SAMPLE_PROBE_BYTES)
|
||||||
|
if pos + 8 > len(head):
|
||||||
|
return None
|
||||||
|
size, kind = struct.unpack(">I4s", head[pos : pos + 8])
|
||||||
|
if size < 8:
|
||||||
|
return None
|
||||||
|
if kind != b"sidx":
|
||||||
|
pos += size
|
||||||
|
continue
|
||||||
|
read_to(pos + size)
|
||||||
|
timescale, subsegments = parse_sidx(head[pos : pos + size], pos)
|
||||||
|
if not subsegments:
|
||||||
|
return None
|
||||||
|
kept = subsegments[self.sample_slice([start for start, _, _ in subsegments], timescale, config.sample)]
|
||||||
|
self.sample_bytes = head[:pos], f"{kept[0][1]}-{kept[-1][2]}"
|
||||||
|
return self.sample_bytes
|
||||||
|
return None
|
||||||
|
|
||||||
|
def can_sample(self, session: Union[Session, "RnetSession"]) -> bool:
|
||||||
|
"""Return False when a sample cannot cut this track, because it is one file with no segment index."""
|
||||||
|
if self.descriptor == self.Descriptor.URL:
|
||||||
|
return bool(self.sample_byte_range(session))
|
||||||
|
if self.descriptor == self.Descriptor.DASH:
|
||||||
|
from unshackle.core.manifests import DASH
|
||||||
|
|
||||||
|
return DASH.has_segment_timing(self)
|
||||||
|
return True
|
||||||
|
|
||||||
def __repr__(self) -> str:
|
def __repr__(self) -> str:
|
||||||
return "{name}({items})".format(
|
return "{name}({items})".format(
|
||||||
@@ -741,8 +844,17 @@ class Track:
|
|||||||
if DOWNLOAD_LICENCE_ONLY.is_set():
|
if DOWNLOAD_LICENCE_ONLY.is_set():
|
||||||
progress(downloaded="[yellow]SKIPPED")
|
progress(downloaded="[yellow]SKIPPED")
|
||||||
else:
|
else:
|
||||||
|
sample_range = None
|
||||||
|
if config.sample and track_type in ("Video", "Audio"):
|
||||||
|
sample_range = self.sample_byte_range(dl_session)
|
||||||
|
if not sample_range:
|
||||||
|
raise click.ClickException(
|
||||||
|
"--sample cannot cut this track: it is one file with no segment index."
|
||||||
|
)
|
||||||
for status_update in self.downloader(
|
for status_update in self.downloader(
|
||||||
urls=self.url,
|
urls=[{"url": self.url, "headers": {"Range": f"bytes={sample_range[1]}"}}]
|
||||||
|
if sample_range
|
||||||
|
else self.url,
|
||||||
output_dir=save_path.parent,
|
output_dir=save_path.parent,
|
||||||
filename=save_path.name,
|
filename=save_path.name,
|
||||||
headers=dl_session.headers,
|
headers=dl_session.headers,
|
||||||
@@ -760,6 +872,14 @@ class Track:
|
|||||||
status_update["downloaded"] = f"URL {downloaded}"
|
status_update["downloaded"] = f"URL {downloaded}"
|
||||||
progress(**status_update)
|
progress(**status_update)
|
||||||
|
|
||||||
|
if sample_range:
|
||||||
|
media_path = save_path.with_name(f"{save_path.name}.media")
|
||||||
|
save_path.replace(media_path)
|
||||||
|
with save_path.open("wb") as joined, media_path.open("rb") as media:
|
||||||
|
joined.write(sample_range[0])
|
||||||
|
shutil.copyfileobj(media, joined)
|
||||||
|
media_path.unlink()
|
||||||
|
|
||||||
self.path = save_path
|
self.path = save_path
|
||||||
events.emit(events.Types.TRACK_DOWNLOADED, track=self)
|
events.emit(events.Types.TRACK_DOWNLOADED, track=self)
|
||||||
|
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ from unshackle.core.utilities import (
|
|||||||
valid_language,
|
valid_language,
|
||||||
)
|
)
|
||||||
from unshackle.core.utils.collections import as_list, flatten
|
from unshackle.core.utils.collections import as_list, flatten
|
||||||
|
from unshackle.core.utils.subprocess import run_step
|
||||||
|
|
||||||
MP4BOX_PROGRESS = re.compile(r"\((\d{1,3})/100\)")
|
MP4BOX_PROGRESS = re.compile(r"\((\d{1,3})/100\)")
|
||||||
MP4BOX_OPEN_FAILED = re.compile(r"error while opening|Error opening file|Invalid IsoMedia File", re.I)
|
MP4BOX_OPEN_FAILED = re.compile(r"error while opening|Error opening file|Invalid IsoMedia File", re.I)
|
||||||
@@ -495,6 +496,11 @@ class Tracks:
|
|||||||
selected.append(track)
|
selected.append(track)
|
||||||
return selected
|
return selected
|
||||||
|
|
||||||
|
@property
|
||||||
|
def sample_base(self) -> float:
|
||||||
|
"""The earliest ``sample_start`` of the video and audio tracks, which a sample mux uses as time zero."""
|
||||||
|
return min((t.sample_start for t in (*self.videos, *self.audio)), default=0.0)
|
||||||
|
|
||||||
def _mux_mp4(
|
def _mux_mp4(
|
||||||
self,
|
self,
|
||||||
title: str,
|
title: str,
|
||||||
@@ -524,6 +530,7 @@ class Tracks:
|
|||||||
progress = progress or partial(lambda **kwargs: None)
|
progress = progress or partial(lambda **kwargs: None)
|
||||||
|
|
||||||
cl = [str(mp4box)]
|
cl = [str(mp4box)]
|
||||||
|
sample_base = self.sample_base
|
||||||
names: list[str] = []
|
names: list[str] = []
|
||||||
cleaned: list[Path] = []
|
cleaned: list[Path] = []
|
||||||
track_id = 0
|
track_id = 0
|
||||||
@@ -557,7 +564,8 @@ class Tracks:
|
|||||||
elif candidate.exists():
|
elif candidate.exists():
|
||||||
candidate.unlink()
|
candidate.unlink()
|
||||||
|
|
||||||
cl.extend(["-add", f"{source}:lang={track.language}"])
|
delay = round((track.sample_start - sample_base) * 1000) if config.sample else 0
|
||||||
|
cl.extend(["-add", f"{source}:lang={track.language}" + (f":delay={delay}" if delay else "")])
|
||||||
name = "" if isinstance(track, Video) else (track.get_track_name() or "")
|
name = "" if isinstance(track, Video) else (track.get_track_name() or "")
|
||||||
if name:
|
if name:
|
||||||
names.extend(["-name", f"{track_id}={name}"])
|
names.extend(["-name", f"{track_id}={name}"])
|
||||||
@@ -568,7 +576,7 @@ class Tracks:
|
|||||||
cl.extend(names)
|
cl.extend(names)
|
||||||
|
|
||||||
chapters_path = None
|
chapters_path = None
|
||||||
if self.chapters:
|
if self.chapters and not config.sample:
|
||||||
chapters_path = config.directories.temp / config.filenames.chapters.format(
|
chapters_path = config.directories.temp / config.filenames.chapters.format(
|
||||||
title=sanitize_filename(title), random=f"{self.chapters.id}_{os.getpid()}_{threading.get_ident()}"
|
title=sanitize_filename(title), random=f"{self.chapters.id}_{os.getpid()}_{threading.get_ident()}"
|
||||||
)
|
)
|
||||||
@@ -632,6 +640,19 @@ class Tracks:
|
|||||||
returncode = p.wait()
|
returncode = p.wait()
|
||||||
if returncode < 2 and any(MP4BOX_OPEN_FAILED.search(line) for line in errors):
|
if returncode < 2 and any(MP4BOX_OPEN_FAILED.search(line) for line in errors):
|
||||||
returncode = 2
|
returncode = 2
|
||||||
|
if config.sample and returncode == 0:
|
||||||
|
start, end = (seconds - sample_base for seconds in config.sample)
|
||||||
|
cut_path = output_path.with_suffix(".cut.mp4")
|
||||||
|
try:
|
||||||
|
run_step(
|
||||||
|
[mp4box, "-splitx", f"{start:.3f}:{end:.3f}", output_path, "-out", cut_path],
|
||||||
|
output=cut_path,
|
||||||
|
label="MP4Box sample cut",
|
||||||
|
)
|
||||||
|
os.replace(cut_path, output_path)
|
||||||
|
except RuntimeError as e:
|
||||||
|
returncode = 2
|
||||||
|
errors.append(f"MP4Box could not cut the sample window: {e}")
|
||||||
mux_duration_ms = round((time.monotonic() - mux_start_time) * 1000, 1)
|
mux_duration_ms = round((time.monotonic() - mux_start_time) * 1000, 1)
|
||||||
output_size = output_path.stat().st_size if output_path.exists() else 0
|
output_size = output_path.stat().st_size if output_path.exists() else 0
|
||||||
|
|
||||||
@@ -729,6 +750,16 @@ class Tracks:
|
|||||||
"--no-date", # remove dates from the output for security
|
"--no-date", # remove dates from the output for security
|
||||||
]
|
]
|
||||||
|
|
||||||
|
sample_base = self.sample_base
|
||||||
|
|
||||||
|
def sample_sync(start: float) -> list[str]:
|
||||||
|
shift = round((start - sample_base) * 1000)
|
||||||
|
return ["--sync", f"0:{shift}"] if config.sample and shift else []
|
||||||
|
|
||||||
|
if config.sample:
|
||||||
|
start, end = (seconds - sample_base for seconds in config.sample)
|
||||||
|
cl.extend(["--split", f"parts:{start:.3f}s-{end:.3f}s"])
|
||||||
|
|
||||||
if config.muxing.get("set_title", True):
|
if config.muxing.get("set_title", True):
|
||||||
cl.extend(["--title", title])
|
cl.extend(["--title", title])
|
||||||
|
|
||||||
@@ -782,6 +813,7 @@ class Tracks:
|
|||||||
f"0:{vt.is_original_lang}",
|
f"0:{vt.is_original_lang}",
|
||||||
"--compression",
|
"--compression",
|
||||||
"0:none",
|
"0:none",
|
||||||
|
*sample_sync(vt.sample_start),
|
||||||
]
|
]
|
||||||
|
|
||||||
# Add FPS fix if needed (typically for hybrid mode to prevent sync issues)
|
# Add FPS fix if needed (typically for hybrid mode to prevent sync issues)
|
||||||
@@ -839,6 +871,7 @@ class Tracks:
|
|||||||
f"0:{at.is_original_lang}",
|
f"0:{at.is_original_lang}",
|
||||||
"--compression",
|
"--compression",
|
||||||
"0:none",
|
"0:none",
|
||||||
|
*sample_sync(at.sample_start),
|
||||||
"(",
|
"(",
|
||||||
str(at.path),
|
str(at.path),
|
||||||
")",
|
")",
|
||||||
@@ -872,6 +905,7 @@ class Tracks:
|
|||||||
f"0:{st.is_original_lang}",
|
f"0:{st.is_original_lang}",
|
||||||
"--compression",
|
"--compression",
|
||||||
"0:none",
|
"0:none",
|
||||||
|
*sample_sync(st.sample_start),
|
||||||
"(",
|
"(",
|
||||||
str(st.path),
|
str(st.path),
|
||||||
")",
|
")",
|
||||||
@@ -883,6 +917,8 @@ class Tracks:
|
|||||||
title=sanitize_filename(title), random=f"{self.chapters.id}_{os.getpid()}_{threading.get_ident()}"
|
title=sanitize_filename(title), random=f"{self.chapters.id}_{os.getpid()}_{threading.get_ident()}"
|
||||||
)
|
)
|
||||||
self.chapters.dump(chapters_path, fallback_name=config.chapter_fallback_name)
|
self.chapters.dump(chapters_path, fallback_name=config.chapter_fallback_name)
|
||||||
|
if config.sample and sample_base:
|
||||||
|
cl.extend(["--chapter-sync", str(round(-sample_base * 1000))])
|
||||||
cl.extend(["--chapter-charset", "UTF-8", "--chapters", str(chapters_path)])
|
cl.extend(["--chapter-charset", "UTF-8", "--chapters", str(chapters_path)])
|
||||||
else:
|
else:
|
||||||
chapters_path = None
|
chapters_path = None
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ from unshackle.core.utils.subprocess import log_tool_run
|
|||||||
|
|
||||||
|
|
||||||
class Video(Track):
|
class Video(Track):
|
||||||
|
sample_tail = 1
|
||||||
|
|
||||||
class Codec(str, Enum):
|
class Codec(str, Enum):
|
||||||
AVC = "H.264"
|
AVC = "H.264"
|
||||||
HEVC = "H.265"
|
HEVC = "H.265"
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import click
|
|||||||
from click.shell_completion import CompletionItem
|
from click.shell_completion import CompletionItem
|
||||||
from pywidevine.cdm import Cdm as WidevineCdm
|
from pywidevine.cdm import Cdm as WidevineCdm
|
||||||
|
|
||||||
|
from unshackle.core.constants import DEFAULT_SAMPLE
|
||||||
from unshackle.core.tracks.audio import Audio
|
from unshackle.core.tracks.audio import Audio
|
||||||
from unshackle.core.tracks.subtitle import Subtitle
|
from unshackle.core.tracks.subtitle import Subtitle
|
||||||
from unshackle.core.tracks.video import Video
|
from unshackle.core.tracks.video import Video
|
||||||
@@ -505,6 +506,26 @@ class SlowDelayRange(click.ParamType):
|
|||||||
return (low, high)
|
return (low, high)
|
||||||
|
|
||||||
|
|
||||||
|
class SampleRange(click.ParamType):
|
||||||
|
"""Parses a sample window string like '60-120' into a tuple of (start, end) seconds."""
|
||||||
|
|
||||||
|
name = "sample_range"
|
||||||
|
|
||||||
|
def convert(
|
||||||
|
self, value: Any, param: Optional[click.Parameter], ctx: Optional[click.Context]
|
||||||
|
) -> Optional[tuple[int, int]]:
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return DEFAULT_SAMPLE if value else None
|
||||||
|
if isinstance(value, (list, tuple)) and len(value) == 2:
|
||||||
|
value = f"{value[0]}-{value[1]}"
|
||||||
|
|
||||||
|
match = re.match(r"^(\d+)-(\d+)$", str(value))
|
||||||
|
if not match or int(match.group(1)) >= int(match.group(2)):
|
||||||
|
self.fail(f"'{value}' is not a valid sample window. Use format: START-END (e.g., 60-120)", param, ctx)
|
||||||
|
|
||||||
|
return (int(match.group(1)), int(match.group(2)))
|
||||||
|
|
||||||
|
|
||||||
SEASON_RANGE = SeasonRange()
|
SEASON_RANGE = SeasonRange()
|
||||||
LANGUAGE_RANGE = LanguageRange()
|
LANGUAGE_RANGE = LanguageRange()
|
||||||
QUALITY_LIST = QualityList()
|
QUALITY_LIST = QualityList()
|
||||||
@@ -512,3 +533,4 @@ AUDIO_CODEC_LIST = AudioCodecList(Audio.Codec)
|
|||||||
VIDEO_CODEC_LIST = MultipleVideoCodecChoice(Video.Codec)
|
VIDEO_CODEC_LIST = MultipleVideoCodecChoice(Video.Codec)
|
||||||
SUBTITLE_CODEC = SubtitleCodecChoice(Subtitle.Codec)
|
SUBTITLE_CODEC = SubtitleCodecChoice(Subtitle.Codec)
|
||||||
SLOW_DELAY_RANGE = SlowDelayRange()
|
SLOW_DELAY_RANGE = SlowDelayRange()
|
||||||
|
SAMPLE_RANGE = SampleRange()
|
||||||
|
|||||||
Reference in New Issue
Block a user