From c119e6f6448a64c2de514dfedad9b29c62e2253a Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Tue, 3 Oct 2023 11:46:42 -0700 Subject: [PATCH] [BUGFIX] Fix match-filter edge case and date_range.before (#750) Fixes yet another bug that is match-filter related. The `date_range.before` value would get ignored if you specified any other match-filters. That is fixed by making them `&` behind-the-scenes. --- src/ytdl_sub/config/plugin.py | 9 -- src/ytdl_sub/plugins/match_filters.py | 95 +++++++++---------- .../subscription_ytdl_options.py | 56 +++++++++-- 3 files changed, 94 insertions(+), 66 deletions(-) diff --git a/src/ytdl_sub/config/plugin.py b/src/ytdl_sub/config/plugin.py index 6c3cac710..9ddc083c0 100644 --- a/src/ytdl_sub/config/plugin.py +++ b/src/ytdl_sub/config/plugin.py @@ -72,15 +72,6 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC): Class to define the new plugin functionality """ - @classmethod - def default_ytdl_options(cls) -> Dict: - """ - Returns - ------- - ytdl options to enable if the plugin is not specified in the download - """ - return {} - def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: """ Returns diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index 4e58d31f9..ef4d6d4a7 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -1,11 +1,8 @@ +import copy from typing import Any -from typing import Dict from typing import List -from typing import Optional from typing import Tuple -from yt_dlp import match_filter_func - from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.preset_options import OptionsDictValidator from ytdl_sub.utils.logger import Logger @@ -13,7 +10,50 @@ logger = Logger.get("match_filters") -_DEFAULT_DOWNLOAD_MATCH_FILTERS: List[str] = ["!is_live & !is_upcoming & !post_live"] + +def default_filters() -> Tuple[List[str], List[str]]: + """ + Returns + ------- + Default filters and breaking filters to always use + """ + return ["!is_live & !is_upcoming & !post_live"], [] + + +def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]: + """ + Parameters + ---------- + filters + User-defined match-filters + to_combine + Filters that need to be combined via AND to the original filters. + These are derived from plugins + + Returns + ------- + merged filters + + Raises + ------ + ValueError + Only supports combining 1 filter at this time. Should never be hit by users + """ + if len(to_combine) == 0: + return filters + if not filters: + return copy.deepcopy(to_combine) + + if len(to_combine) > 1: + raise ValueError("Match-filters to combine only supports 1 at this time") + + output_filters: List[str] = [] + filter_to_combine: str = to_combine[0] + + for match_filter in filters: + output_filters.append(f"{match_filter} & {filter_to_combine}") + + return output_filters class MatchFiltersOptions(OptionsDictValidator): @@ -46,24 +86,18 @@ class MatchFiltersOptions(OptionsDictValidator): # - "availability=?public" """ - _optional_keys = {"filters", "download_filters"} + _optional_keys = {"filters"} @classmethod def partial_validate(cls, name: str, value: Any) -> None: """Ensure filters looks right""" if isinstance(value, dict): value["filters"] = value.get("filters", [""]) - value["download_filters"] = value.get("download_filters", [""]) _ = cls(name, value) def __init__(self, name, value): super().__init__(name, value) self._filters = self._validate_key_if_present(key="filters", validator=StringListValidator) - self._download_filters = self._validate_key( - key="download_filters", - validator=StringListValidator, - default=_DEFAULT_DOWNLOAD_MATCH_FILTERS, - ) @property def filters(self) -> List[str]: @@ -74,35 +108,10 @@ def filters(self) -> List[str]: """ return [validator.value for validator in self._filters.list] if self._filters else [] - @property - def download_filters(self) -> List[str]: - """ - Filters to apply during the download stage. This can be useful when building presets - that contain match-filters that you do not want to conflict with metadata-based - match-filters since they act as logical OR's. - - By default, if no download_filters are present, then the filter - ``"!is_live & !is_upcoming & !post_live"`` is added. - """ - return [validator.value for validator in self._download_filters.list] - class MatchFiltersPlugin(Plugin[MatchFiltersOptions]): plugin_options_type = MatchFiltersOptions - @classmethod - def default_ytdl_options(cls) -> Dict: - """ - Returns - ------- - match-filter to filter out live + upcoming videos when downloading - """ - return { - "match_filter": match_filter_func( - filters=[], breaking_filters=_DEFAULT_DOWNLOAD_MATCH_FILTERS - ), - } - def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: """ Returns @@ -110,15 +119,3 @@ def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: match_filters to apply at the metadata stage """ return self.plugin_options.filters, [] - - def ytdl_options(self) -> Optional[Dict]: - """ - Returns - ------- - match_filters to apply at the download stage - """ - return { - "match_filter": match_filter_func( - filters=[], breaking_filters=self.plugin_options.download_filters - ), - } diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index df6fb7423..6a7a7ceae 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -15,12 +15,17 @@ from ytdl_sub.plugins.file_convert import FileConvertPlugin from ytdl_sub.plugins.format import FormatPlugin from ytdl_sub.plugins.match_filters import MatchFiltersPlugin +from ytdl_sub.plugins.match_filters import combine_filters +from ytdl_sub.plugins.match_filters import default_filters from ytdl_sub.plugins.subtitles import SubtitlesPlugin from ytdl_sub.utils.ffmpeg import FFMPEG +from ytdl_sub.utils.logger import Logger from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive PluginT = TypeVar("PluginT", bound=Plugin) +logger = Logger.get("ytdl-options") + class SubscriptionYTDLOptions: def __init__( @@ -89,10 +94,10 @@ def _output_options(self) -> Dict: return ytdl_options def _plugin_ytdl_options(self, plugin: Type[PluginT]) -> Dict: - if not (plugin_obj := self._get_plugin(plugin)): - return plugin.default_ytdl_options() + if plugin_obj := self._get_plugin(plugin): + return plugin_obj.ytdl_options() - return plugin_obj.ytdl_options() + return {} @property def _user_ytdl_options(self) -> Dict: @@ -100,14 +105,50 @@ def _user_ytdl_options(self) -> Dict: @property def _plugin_match_filters(self) -> Dict: - match_filters: List[str] = [] - breaking_match_filters: List[str] = [] + """ + All match-filters from every plugin to fetch metadata. + In order for other plugins to not collide with user-defined match-filters, do + + match_filters = user_match_filters or {} + for plugin in plugins: + for filter in match_filters: + AND plugin's match filters onto the existing filters + + Otherwise, the filters separately act as an OR + """ + match_filters, breaking_match_filters = default_filters() + + match_filters_plugin = self._get_plugin(MatchFiltersPlugin) + if match_filters_plugin: + ( + user_match_filters, + user_breaking_match_filters, + ) = match_filters_plugin.ytdl_options_match_filters() + match_filters = combine_filters(filters=user_match_filters, to_combine=match_filters) + breaking_match_filters = combine_filters( + filters=user_breaking_match_filters, to_combine=breaking_match_filters + ) + for plugin in self._plugins: + # Do not re-add original match-filters plugin + if isinstance(plugin, MatchFiltersPlugin): + continue + pl_match_filters, pl_breaking_match_filters = plugin.ytdl_options_match_filters() - match_filters.extend(pl_match_filters) - breaking_match_filters.extend(pl_breaking_match_filters) + match_filters = combine_filters(filters=match_filters, to_combine=pl_match_filters) + breaking_match_filters = combine_filters( + filters=breaking_match_filters, to_combine=pl_breaking_match_filters + ) + logger.debug( + "Setting match-filters: %s", + "\n - ".join([""] + match_filters) if match_filters else "[]", + ) + logger.debug( + "Setting breaking-match-filters: %s", + "\n - ".join([""] + breaking_match_filters) if breaking_match_filters else "[]", + ) return { "match_filter": match_filter_func( filters=match_filters, breaking_filters=breaking_match_filters @@ -145,7 +186,6 @@ def download_builder(self) -> YTDLOptionsBuilder: self._plugin_ytdl_options(ChaptersPlugin), self._plugin_ytdl_options(AudioExtractPlugin), self._plugin_ytdl_options(FormatPlugin), - self._plugin_ytdl_options(MatchFiltersPlugin), self._user_ytdl_options, # user ytdl options... self._download_only_options, # then download_only options )