[FEATURE] Multi-url support for YouTube music presets (#1113)

Adds support to specify multiple URLs for the YouTube-based music presets. More info here: https://ytdl-sub.readthedocs.io/en/latest/prebuilt_presets/music.html

docs/source/prebuilt_presets/music.rst

@@ -6,6 +6,12 @@ Music downloadable by yt-dlp comes in many flavors. ``ytdl-sub`` offers a suite
 of various presets for handling some of the most popular forms of uploaded music
 content.
 
+.. hint::
+
+  The subscription *value* (denoted by =) will set the genre tag for all music scraped under its key
+  for all music presets.
+
+
 YouTube Releases
 ----------------
 Many artists, especially those auto-uploaded as ``Topics`` in YouTube have a section on
@@ -18,11 +24,18 @@ Playlists are recognized as the album, and videos within it are tracks.
 
    YouTube Releases:
      = Jazz:  # Sets genre tag to "Jazz"
-       "Thelonious Monk": "https://www.youtube.com/@theloniousmonk3870/releases"
+       "Thelonious Monk": "https://www.youtube.com/@officialtheloniousmonk/releases"
 
-.. hint::
+If you are only interested in a subset of albums, you can provide their playlists as separate values in the form
+of an array, like so:
+
+.. code-block:: yaml
 
-  The subscription *value* (denoted by =) will set the genre tag for all music scraped under its key.
+   YouTube Releases:
+     = Jazz:
+       "Thelonious Monk":
+         - "https://www.youtube.com/playlist?list=OLAK5uy_lcqINwfzkw73TPnAt6MlpB6V0gM9VzQu8"  # Monk on Monk
+         - "https://www.youtube.com/playlist?list=OLAK5uy_nhuvjuZOO3yLIWCbQzbiWfyzkGapSIuYw"  # Late Night Thelonious Monk
 
 YouTube Full Albums
 -------------------
@@ -38,6 +51,17 @@ Videos are recognized as the album, and chapters within it are tracks.
      = Lofi:
        "Game Chops": "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i"
 
+If you are only interested in a subset of albums, you can provide their video as separate values in the form
+of an array, like so:
+
+.. code-block:: yaml
+
+   YouTube Full Albums:
+     = Lofi:
+       "Game Chops":
+         - "https://www.youtube.com/watch?v=m7vBrD7LMLI"  # Zelda & Sleep Ensemble Collection
+         - "https://www.youtube.com/watch?v=w0XebCwSpKI"  # Study Buddy ~ video game lofi mix
+
 Soundcloud Discography
 ----------------------
 SoundCloud tracks can be uploaded as either a single, part of an album, or a collaboration

src/ytdl_sub/downloaders/url/downloader.py

@@ -462,10 +462,13 @@ def _download_metadata(self, url: str, validator: UrlValidator) -> Iterable[Entr
         download_reversed = ScriptUtils.bool_formatter_output(
             self.overrides.apply_formatter(validator.download_reverse)
         )
+        include_sibling_metadata = ScriptUtils.bool_formatter_output(
+            self.overrides.apply_formatter(validator.include_sibling_metadata)
+        )
 
         parents, orphan_entries = self._download_url_metadata(
             url=url,
-            include_sibling_metadata=validator.include_sibling_metadata,
+            include_sibling_metadata=include_sibling_metadata,
             ytdl_options_overrides=metadata_ytdl_options,
         )
 

src/ytdl_sub/downloaders/url/validators.py

@@ -12,7 +12,6 @@
 from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
-from ytdl_sub.validators.validators import BoolValidator
 from ytdl_sub.validators.validators import ListValidator
 
 
@@ -86,7 +85,9 @@ def __init__(self, name, value):
             key="ytdl_options", validator=YTDLOptions, default={}
         )
         self._include_sibling_metadata = self._validate_key(
-            key="include_sibling_metadata", validator=BoolValidator, default=False
+            key="include_sibling_metadata",
+            validator=OverridesBooleanFormatterValidator,
+            default="False",
         )
 
     @property
@@ -170,14 +171,14 @@ def ytdl_options(self) -> YTDLOptions:
         return self._ytdl_options
 
     @property
-    def include_sibling_metadata(self) -> bool:
+    def include_sibling_metadata(self) -> OverridesBooleanFormatterValidator:
         """
         Optional. Whether to include sibling metadata as an entry variable, which comprises basic
         metadata from all other entries (including itself) that belong to the same playlist. For
         channels or large playlists, this becomes memory-intensive since you are storing
         ``n^2`` metadata. Defaults to False.
         """
-        return self._include_sibling_metadata.value
+        return self._include_sibling_metadata
 
 
 class UrlStringOrDictValidator(UrlValidator):

src/ytdl_sub/prebuilt_presets/helpers/url.yaml

@@ -25,109 +25,209 @@ presets:
             uid: "avatar_uncropped"
           - name: "{banner_uncropped_thumbnail_file_name}"
             uid: "banner_uncropped"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url2}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url3}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url4}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url5}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url6}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url7}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url8}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url9}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url10}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url11}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url12}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url13}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url14}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url15}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url16}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url17}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url18}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url19}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url20}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url21}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url22}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url23}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url24}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url25}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url26}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url27}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url28}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url29}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url30}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url31}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url32}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url33}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url34}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url35}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url36}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url37}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url38}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url39}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url40}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url41}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url42}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url43}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url44}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url45}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url46}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url47}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url48}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url49}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url50}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url51}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url52}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url53}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url54}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url55}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url56}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url57}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url58}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url59}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url60}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url61}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url62}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url63}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url64}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url65}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url66}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url67}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url68}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url69}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url70}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url71}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url72}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url73}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url74}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url75}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url76}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url77}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url78}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url79}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url80}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url81}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url82}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url83}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url84}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url85}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url86}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url87}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url88}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url89}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url90}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url91}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url92}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url93}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url94}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url95}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url96}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url97}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url98}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url99}"
+        include_sibling_metadata: "{include_sibling_metadata}"
       - url: "{url100}"
-
+        include_sibling_metadata: "{include_sibling_metadata}"
     overrides:
       avatar_uncropped_thumbnail_file_name: ""
       banner_uncropped_thumbnail_file_name: ""
+      include_sibling_metadata: False
 
       subscription_array: "{ [] }"
       subscription_value: ""

src/ytdl_sub/prebuilt_presets/music/albums_from_chapters.yaml

@@ -2,14 +2,18 @@ presets:
 
   # assumes that each chapter in an entry is a song
   _albums_from_chapters:
-    preset: "Single"
+    preset:
+      - "_music_base"
+      - "_multi_url"
 
     chapters:
       embed_chapters: True
     split_by_chapters:
       when_no_chapters: "pass"
 
     overrides:
+      include_sibling_metadata: False
+
       track_title: "{chapter_title}"  # Chapter title is the track title
       track_album: "{title}"  # Video's title is the album title
       track_number: "{chapter_index}"

src/ytdl_sub/prebuilt_presets/music/albums_from_playlists.yaml

@@ -4,12 +4,11 @@ presets:
   _albums_from_playlists:
     preset:
       - "_music_base"
-
-    download:
-      - url: "{url}"
-        include_sibling_metadata: True
+      - "_multi_url"
 
     overrides:
+      include_sibling_metadata: True
+
       track_album: "{playlist_title}"
       track_number: "{playlist_index}"
       track_number_padded: "{playlist_index_padded}"

tests/integration/prebuilt_presets/test_music.py

@@ -1,11 +1,13 @@
 import pytest
 from expected_download import assert_expected_downloads
 from expected_transaction_log import assert_transaction_log_matches
+from integration.plugins.conftest import mock_chapters_class
 
 from ytdl_sub.prebuilt_presets.music import MusicPresets
 from ytdl_sub.subscriptions.subscription import Subscription
 
 
+@pytest.mark.usefixtures(mock_chapters_class.__name__)
 @pytest.mark.parametrize("music_preset", MusicPresets.preset_names)
 class TestPrebuiltMusicPresets:
 
@@ -29,16 +31,19 @@ def test_presets_run(
             },
         }
 
+        num_urls = 1
+        if music_preset in {"YouTube Releases", "YouTube Full Albums"}:
+            preset_dict["overrides"]["url2"] = "https://your.name.here.2"
+            num_urls = 2
+        elif music_preset == "SoundCloud Discography":
+            num_urls = 2  # simulate albums + tracks
+
         subscription = Subscription.from_dict(
             config=config,
             preset_name=subscription_name,
             preset_dict=preset_dict,
         )
 
-        num_urls = 1
-        if music_preset == "SoundCloud Discography":
-            num_urls = 2  # simulate /albums and /tracks
-
         with mock_download_collection_entries(
             is_youtube_channel=False, num_urls=num_urls, is_extracted_audio=True
         ):