feat(media): build yt-dlp and aria2c options
Only flags that actually change aria2's behaviour are sent. yt-dlp already passes -x16 -j16 -s16 --min-split-size 1M, and -x16 is aria2's per-server cap, so re-sending connection counts would be a no-op. What is added: a retry-wait (aria2 defaults to 0 and burns all five tries in under a second), shorter timeouts, a 64M disk cache for sixteen out-of-order writers, falloc on filesystems where it is instant (this one is btrfs), a file log because --console-log-level=warn leaves no forensics, and --lowest-speed-limit, which is off by default and is what stops a wedged connection hanging forever. --continue is deliberately never passed: aria2 writes segments out of order, so resuming from a .part's length would read a sparse file as a valid prefix and silently corrupt the video. external_downloader is mapped to exactly the four protocols Aria2cFD supports rather than "default", so HLS and DASH visibly fall through to the native downloader and concurrent_fragment_downloads is what matters there. noplaylist defaults to true so a video URL carrying &list= stays one video, which is the opposite of yt-dlp's own default. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,7 @@ from ccn_transcribe import errors
|
||||
from ccn_transcribe.backends import openvino_backend as ovb
|
||||
from ccn_transcribe.backends import probe as probe_mod
|
||||
from ccn_transcribe.backends.base import TranscribeRequest
|
||||
from ccn_transcribe.media import ffmpeg
|
||||
|
||||
LIVE = pytest.mark.skipif(
|
||||
not os.environ.get("CCN_LIVE"), reason="set CCN_LIVE=1 to run against the real GPU"
|
||||
@@ -272,8 +273,6 @@ class TestLoadHints:
|
||||
@LIVE
|
||||
class TestLive:
|
||||
def test_real_transcription_on_the_default_device(self, sine_wav: Path) -> None:
|
||||
from ccn_transcribe.media import ffmpeg
|
||||
|
||||
backend = ovb.OpenvinoGpuBackend()
|
||||
backend.load()
|
||||
result = backend.transcribe(ffmpeg.decode_16k_mono(sine_wav), TranscribeRequest())
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from ccn_transcribe.media import ytdlp_opts as opts
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class TestAria2cArgs:
|
||||
def test_sets_a_retry_wait(self) -> None:
|
||||
# aria2's default is 0, so it burns all five tries in under a second
|
||||
# against a struggling server.
|
||||
assert "--retry-wait=3" in opts.aria2c_args()
|
||||
|
||||
def test_enables_the_stall_abort(self) -> None:
|
||||
# --lowest-speed-limit defaults to 0, meaning a wedged connection hangs
|
||||
# forever. This is the single biggest real-world hang fix.
|
||||
assert any(a.startswith("--lowest-speed-limit=") for a in opts.aria2c_args())
|
||||
|
||||
def test_stall_abort_can_be_disabled_for_genuinely_slow_links(self) -> None:
|
||||
assert "--lowest-speed-limit=0" in opts.aria2c_args(lowest_speed="0")
|
||||
|
||||
def test_raises_the_disk_cache(self) -> None:
|
||||
assert "--disk-cache=64M" in opts.aria2c_args()
|
||||
|
||||
def test_shortens_the_timeouts(self) -> None:
|
||||
args = opts.aria2c_args()
|
||||
assert "--timeout=30" in args
|
||||
assert "--connect-timeout=15" in args
|
||||
|
||||
def test_writes_a_log_when_asked(self, tmp_path: Path) -> None:
|
||||
args = opts.aria2c_args(log_file=tmp_path / "aria2.log")
|
||||
assert f"--log={tmp_path / 'aria2.log'}" in args
|
||||
assert "--log-level=notice" in args
|
||||
|
||||
def test_omits_the_log_when_not_asked(self) -> None:
|
||||
assert not any(a.startswith("--log=") for a in opts.aria2c_args())
|
||||
|
||||
def test_never_passes_continue(self) -> None:
|
||||
# aria2 writes segments out of order, so --continue would resume from the
|
||||
# file length over a sparse file and silently corrupt the download.
|
||||
args = opts.aria2c_args()
|
||||
assert not any(a.startswith(("--continue", "-c")) for a in args)
|
||||
|
||||
def test_does_not_repeat_flags_yt_dlp_already_locks(self) -> None:
|
||||
# These are appended after user args and cannot be overridden; passing
|
||||
# them is a silent no-op that only confuses the next reader.
|
||||
args = " ".join(opts.aria2c_args())
|
||||
for locked in (
|
||||
"--allow-overwrite",
|
||||
"--always-resume",
|
||||
"--auto-file-renaming",
|
||||
"--force-save",
|
||||
):
|
||||
assert locked not in args
|
||||
|
||||
def test_does_not_restate_aria2_defaults(self) -> None:
|
||||
# --max-tries=5 is already aria2's default; yt-dlp does not set it.
|
||||
assert not any(a.startswith("--max-tries") for a in opts.aria2c_args())
|
||||
|
||||
def test_connection_count_is_only_sent_when_it_differs(self) -> None:
|
||||
# -x16 is already yt-dlp's default and aria2's per-server cap.
|
||||
assert not any(a.startswith("-x") for a in opts.aria2c_args())
|
||||
assert "-x4" in opts.aria2c_args(connections=4)
|
||||
assert "-s4" in opts.aria2c_args(connections=4)
|
||||
|
||||
def test_user_arguments_sort_last_so_they_win(self) -> None:
|
||||
# aria2 takes the last occurrence of a repeated flag.
|
||||
args = opts.aria2c_args(extra=["--disk-cache=8M"])
|
||||
assert args[-1] == "--disk-cache=8M"
|
||||
|
||||
def test_file_allocation_is_falloc_on_supporting_filesystems(self) -> None:
|
||||
assert "--file-allocation=falloc" in opts.aria2c_args(file_allocation="falloc")
|
||||
|
||||
def test_file_allocation_falls_back_to_none(self) -> None:
|
||||
assert "--file-allocation=none" in opts.aria2c_args(file_allocation="none")
|
||||
|
||||
|
||||
class TestAllocationDetection:
|
||||
@pytest.mark.parametrize("fstype", ["btrfs", "ext4", "xfs"])
|
||||
def test_supported_filesystems_get_falloc(self, fstype: str) -> None:
|
||||
assert opts.allocation_for(fstype) == "falloc"
|
||||
|
||||
@pytest.mark.parametrize("fstype", ["nfs", "tmpfs", "vfat", "unknown"])
|
||||
def test_other_filesystems_fall_back(self, fstype: str) -> None:
|
||||
assert opts.allocation_for(fstype) == "none"
|
||||
|
||||
def test_detects_this_machines_filesystem(self, tmp_path: Path) -> None:
|
||||
# /home here is btrfs, which is exactly why falloc matters.
|
||||
assert opts.filesystem_type(tmp_path) is not None
|
||||
|
||||
|
||||
class TestYdlOpts:
|
||||
def test_remuxes_to_mkv_rather_than_re_encoding(self) -> None:
|
||||
# This ffmpeg build has no x264/x265 encoders; mkv accepts any codec pair
|
||||
# so the merge stays a stream copy.
|
||||
assert opts.ydl_opts("x")["merge_output_format"] == "mkv"
|
||||
|
||||
def test_resumes_partial_downloads(self) -> None:
|
||||
assert opts.ydl_opts("x")["continuedl"] is True
|
||||
|
||||
def test_writes_the_info_json(self) -> None:
|
||||
assert opts.ydl_opts("x")["writeinfojson"] is True
|
||||
|
||||
def test_maps_aria2c_to_the_protocols_it_actually_supports(self) -> None:
|
||||
# Aria2cFD.SUPPORTED_PROTOCOLS is http/https/ftp/ftps only; an explicit
|
||||
# map avoids relying on yt-dlp's silent fallback for HLS and DASH.
|
||||
got = opts.ydl_opts("x", opts.DownloadConfig(use_aria2c=True))["external_downloader"]
|
||||
assert set(got) == {"http", "https", "ftp", "ftps"}
|
||||
assert set(got.values()) == {"aria2c"}
|
||||
|
||||
def test_omits_the_external_downloader_when_aria2c_is_absent(self) -> None:
|
||||
assert "external_downloader" not in opts.ydl_opts(
|
||||
"x", opts.DownloadConfig(use_aria2c=False)
|
||||
)
|
||||
|
||||
def test_raises_concurrent_fragments_for_hls_where_aria2c_cannot_help(self) -> None:
|
||||
assert opts.ydl_opts("x")["concurrent_fragment_downloads"] == 16
|
||||
|
||||
def test_does_not_expand_a_playlist_by_default(self) -> None:
|
||||
# A video URL carrying &list= should transcribe that one video.
|
||||
assert opts.ydl_opts("x")["noplaylist"] is True
|
||||
|
||||
def test_playlist_expansion_is_opt_in(self) -> None:
|
||||
assert opts.ydl_opts("x", opts.DownloadConfig(playlist=True))["noplaylist"] is False
|
||||
|
||||
def test_height_cap_is_applied_to_the_format_selector(self) -> None:
|
||||
assert "height<=720" in opts.ydl_opts("x", opts.DownloadConfig(max_height=720))["format"]
|
||||
|
||||
def test_format_prefers_progressive_https_for_aria2c_eligibility(self) -> None:
|
||||
assert "proto:https" in " ".join(opts.ydl_opts("x")["format_sort"])
|
||||
|
||||
def test_cookies_from_browser_is_passed_through(self) -> None:
|
||||
got = opts.ydl_opts("x", opts.DownloadConfig(cookies_from_browser="firefox"))
|
||||
assert got["cookiesfrombrowser"] == ("firefox",)
|
||||
|
||||
def test_cookies_absent_when_not_requested(self) -> None:
|
||||
assert "cookiesfrombrowser" not in opts.ydl_opts("x")
|
||||
|
||||
def test_aria2c_args_are_attached_under_the_executable_name(self, tmp_path: Path) -> None:
|
||||
got = opts.ydl_opts("x", opts.DownloadConfig(use_aria2c=True), aria2_log=tmp_path / "a.log")
|
||||
assert "aria2c" in got["external_downloader_args"]
|
||||
assert any("--retry-wait" in a for a in got["external_downloader_args"]["aria2c"])
|
||||
|
||||
|
||||
class TestJsRuntime:
|
||||
def test_defaults_to_deno_which_ships_in_the_venv(self) -> None:
|
||||
# yt-dlp's own default is {'deno': {}}, and the deno extra puts the binary
|
||||
# in the venv, so cron sees it without any PATH setup.
|
||||
assert "deno" in opts.js_runtime_spec()
|
||||
|
||||
def test_an_explicit_runtime_is_honoured(self) -> None:
|
||||
assert "node" in opts.js_runtime_spec("node")
|
||||
|
||||
def test_node_is_resolved_to_an_absolute_path_when_found(self) -> None:
|
||||
spec = opts.js_runtime_spec("node")
|
||||
path = spec["node"].get("path")
|
||||
if path is not None:
|
||||
assert path.startswith("/")
|
||||
|
||||
def test_an_unknown_runtime_is_rejected(self) -> None:
|
||||
with pytest.raises(Exception, match="runtime"):
|
||||
opts.js_runtime_spec("rhino")
|
||||
|
||||
def test_the_runtime_is_wired_into_ydl_opts(self) -> None:
|
||||
assert "js_runtimes" in opts.ydl_opts("x")
|
||||
|
||||
|
||||
def test_nvm_only_node_is_flagged_as_invisible_to_cron(tmp_path: Path) -> None:
|
||||
nvm = tmp_path / ".nvm/versions/node/v24.16.0/bin/node"
|
||||
nvm.parent.mkdir(parents=True)
|
||||
nvm.write_text("")
|
||||
assert opts.is_nvm_path(str(nvm)) is True
|
||||
assert opts.is_nvm_path("/usr/bin/node") is False
|
||||
|
||||
|
||||
class TestFilesystemDetection:
|
||||
def test_picks_the_deepest_matching_mount(self, tmp_path: Path) -> None:
|
||||
mounts = tmp_path / "mounts"
|
||||
mounts.write_text(f"/dev/a / ext4 rw 0 0\n/dev/b {tmp_path} btrfs rw 0 0\n")
|
||||
assert opts.filesystem_type(tmp_path, mounts) == "btrfs"
|
||||
|
||||
def test_malformed_lines_are_skipped(self, tmp_path: Path) -> None:
|
||||
mounts = tmp_path / "mounts"
|
||||
mounts.write_text(f"garbage\n/dev/b {tmp_path} xfs rw 0 0\n")
|
||||
assert opts.filesystem_type(tmp_path, mounts) == "xfs"
|
||||
|
||||
def test_none_when_the_mount_table_is_unreadable(self, tmp_path: Path) -> None:
|
||||
assert opts.filesystem_type(tmp_path, tmp_path / "absent") is None
|
||||
|
||||
def test_none_when_nothing_matches(self, tmp_path: Path) -> None:
|
||||
mounts = tmp_path / "mounts"
|
||||
mounts.write_text("/dev/a /nowhere-at-all ext4 rw 0 0\n")
|
||||
assert opts.filesystem_type(tmp_path, mounts) is None
|
||||
|
||||
|
||||
class TestNodeDiscovery:
|
||||
def test_prefers_node_on_path(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def on_path(_n: str, _mode: int = 0, _path: str | None = None) -> str:
|
||||
return "/usr/bin/node"
|
||||
|
||||
monkeypatch.setattr(shutil, "which", on_path)
|
||||
assert opts.find_node() == "/usr/bin/node"
|
||||
|
||||
def test_falls_back_to_nvm(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
def absent(_n: str, _mode: int = 0, _path: str | None = None) -> str | None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(shutil, "which", absent)
|
||||
node = tmp_path / ".nvm/versions/node/v24.16.0/bin/node"
|
||||
node.parent.mkdir(parents=True)
|
||||
node.write_text("")
|
||||
assert opts.find_node(tmp_path) == str(node)
|
||||
|
||||
def test_picks_the_newest_nvm_version(
|
||||
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
def absent(_n: str, _mode: int = 0, _path: str | None = None) -> str | None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(shutil, "which", absent)
|
||||
for version in ("v18.0.0", "v24.16.0"):
|
||||
node = tmp_path / f".nvm/versions/node/{version}/bin/node"
|
||||
node.parent.mkdir(parents=True)
|
||||
node.write_text("")
|
||||
assert "v24.16.0" in (opts.find_node(tmp_path) or "")
|
||||
|
||||
def test_none_when_node_is_nowhere(
|
||||
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
def absent(_n: str, _mode: int = 0, _path: str | None = None) -> str | None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(shutil, "which", absent)
|
||||
assert opts.find_node(tmp_path) is None
|
||||
|
||||
def test_node_spec_is_empty_when_node_is_absent(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def no_node(*_a: object, **_k: object) -> str | None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(opts, "find_node", no_node)
|
||||
assert opts.js_runtime_spec("node") == {"node": {}}
|
||||
|
||||
|
||||
def test_a_shallower_mount_does_not_override_a_deeper_one(tmp_path: Path) -> None:
|
||||
# The longest matching mount point wins, whatever order the table lists them.
|
||||
mounts = tmp_path / "mounts"
|
||||
mounts.write_text(f"/dev/b {tmp_path} btrfs rw 0 0\n/dev/a / ext4 rw 0 0\n")
|
||||
assert opts.filesystem_type(tmp_path, mounts) == "btrfs"
|
||||
Reference in New Issue
Block a user