[FEATURE] Persisted subscription logs (#512)

docs/config.rst

@@ -29,7 +29,25 @@ and subscriptions.
 .. autoclass:: ytdl_sub.config.config_validator.ConfigOptions()
   :members:
   :member-order: bysource
+  :exclude-members: persist_logs
 
+persist_logs
+""""""""""""
+Within ``configuration``, define whether logs from subscription downloads
+should be persisted.
+
+.. code-block:: yaml
+
+   configuration:
+     persist_logs:
+       logs_directory: "/path/to/log/directory"
+
+Log files are stored as
+``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``.
+
+.. autoclass:: ytdl_sub.config.config_validator.PersistLogsValidator()
+  :members:
+  :member-order: bysource
 
 presets
 ^^^^^^^

src/ytdl_sub/cli/main.py

@@ -1,13 +1,18 @@
-import argparse
 import gc
 import sys
+from datetime import datetime
+from pathlib import Path
 from typing import List
+from typing import Optional
 from typing import Tuple
 
+from yt_dlp.utils import sanitize_filename
+
 from ytdl_sub.cli.download_args_parser import DownloadArgsParser
 from ytdl_sub.cli.main_args_parser import parser
 from ytdl_sub.config.config_file import ConfigFile
 from ytdl_sub.subscriptions.subscription import Subscription
+from ytdl_sub.utils.file_handler import FileHandler
 from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
 from ytdl_sub.utils.file_lock import working_directory_lock
 from ytdl_sub.utils.logger import Logger
@@ -19,8 +24,41 @@
 _VIEW_EXTRA_ARGS_FORMATTER = "--preset _view --overrides.url {}"
 
 
+def _maybe_write_subscription_log_file(
+    config: ConfigFile,
+    subscription: Subscription,
+    dry_run: bool,
+    exception: Optional[Exception] = None,
+) -> None:
+    success: bool = exception is None
+
+    # If dry-run, do nothing
+    if dry_run:
+        return
+
+    # If persisting logs is disabled, do nothing
+    if not config.config_options.persist_logs:
+        return
+
+    # If persisting successful logs is disabled, do nothing
+    if success and not config.config_options.persist_logs.keep_successful_logs:
+        return
+
+    log_time = datetime.now().strftime("%Y-%m-%d-%H%M%S")
+    log_subscription_name = sanitize_filename(subscription.name).lower().replace(" ", "_")
+    log_success = "success" if success else "error"
+
+    log_filename = f"{log_time}.{log_subscription_name}.{log_success}.log"
+    persist_log_path = Path(config.config_options.persist_logs.logs_directory) / log_filename
+
+    if not success:
+        Logger.log_exit_exception(exception=exception, log_filepath=persist_log_path)
+
+    FileHandler.copy(Logger.debug_log_filename(), persist_log_path)
+
+
 def _download_subscriptions_from_yaml_files(
-    config: ConfigFile, args: argparse.Namespace
+    config: ConfigFile, subscription_paths: List[str], dry_run: bool
 ) -> List[Tuple[Subscription, FileHandlerTransactionLog]]:
     """
     Downloads all subscriptions from one or many subscription yaml files.
@@ -29,18 +67,20 @@ def _download_subscriptions_from_yaml_files(
     ----------
     config
         Configuration file
-    args
-        Arguments from argparse
+    subscription_paths
+        Path to subscription files to download
+    dry_run
+        Whether to dry run or not
 
     Returns
     -------
     List of (subscription, transaction_log)
 
     Raises
     ------
-    Validation exception if main arg is specified as a subscription path
+    Exception
+        Any exception during download
     """
-    subscription_paths: List[str] = args.subscription_paths
     subscriptions: List[Subscription] = []
     output: List[Tuple[Subscription, FileHandlerTransactionLog]] = []
 
@@ -51,15 +91,26 @@ def _download_subscriptions_from_yaml_files(
     for subscription in subscriptions:
         logger.info(
             "Beginning subscription %s for %s",
-            ("dry run" if args.dry_run else "download"),
+            ("dry run" if dry_run else "download"),
             subscription.name,
         )
         logger.debug("Subscription full yaml:\n%s", subscription.as_yaml())
-        transaction_log = subscription.download(dry_run=args.dry_run)
 
-        output.append((subscription, transaction_log))
-        gc.collect()  # Garbage collect after each subscription download
-        Logger.cleanup()  # Cleanup logger after each successful subscription download
+        try:
+            transaction_log = subscription.download(dry_run=dry_run)
+        except Exception as exc:  # pylint: disable=broad-except
+            _maybe_write_subscription_log_file(
+                config=config, subscription=subscription, dry_run=dry_run, exception=exc
+            )
+            raise
+        else:
+            output.append((subscription, transaction_log))
+            _maybe_write_subscription_log_file(
+                config=config, subscription=subscription, dry_run=dry_run
+            )
+            Logger.cleanup()  # Cleanup logger after each successful subscription download
+        finally:
+            gc.collect()  # Garbage collect after each subscription download
 
     return output
 
@@ -136,7 +187,11 @@ def main() -> List[Tuple[Subscription, FileHandlerTransactionLog]]:
 
     with working_directory_lock(config=config):
         if args.subparser == "sub":
-            transaction_logs = _download_subscriptions_from_yaml_files(config=config, args=args)
+            transaction_logs = _download_subscriptions_from_yaml_files(
+                config=config,
+                subscription_paths=args.subscription_paths,
+                dry_run=args.dry_run,
+            )
 
         # One-off download
         elif args.subparser == "dl":

src/ytdl_sub/config/config_file.py

@@ -1,5 +1,6 @@
 import os
 from typing import Any
+from typing import Dict
 
 from ytdl_sub.config.config_validator import ConfigValidator
 from ytdl_sub.config.preset import Preset
@@ -65,3 +66,11 @@ def from_file_path(cls, config_path: str) -> "ConfigFile":
         """
         config_dict = load_yaml(file_path=config_path)
         return ConfigFile.from_dict(config_dict)
+
+    def as_dict(self) -> Dict[str, Any]:
+        """
+        Returns
+        -------
+        The config in its dict form
+        """
+        return self._value

src/ytdl_sub/config/config_validator.py

@@ -3,12 +3,14 @@
 from typing import Optional
 
 from mergedeep import mergedeep
+from yt_dlp.utils import datetime_from_str
 
 from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS
 from ytdl_sub.utils.system import IS_WINDOWS
 from ytdl_sub.validators.file_path_validators import FFmpegFileValidator
 from ytdl_sub.validators.file_path_validators import FFprobeFileValidator
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
+from ytdl_sub.validators.validators import BoolValidator
 from ytdl_sub.validators.validators import LiteralDictValidator
 from ytdl_sub.validators.validators import StringValidator
 
@@ -22,9 +24,71 @@
     _DEFAULT_FFPROBE_PATH = "/usr/bin/ffprobe"
 
 
+class PersistLogsValidator(StrictDictValidator):
+    _required_keys = {"logs_directory"}
+    _optional_keys = {"keep_logs_after", "keep_successful_logs"}
+
+    def __init__(self, name: str, value: Any):
+        super().__init__(name, value)
+
+        self._logs_directory = self._validate_key(key="logs_directory", validator=StringValidator)
+
+        self._keep_logs_after: Optional[str] = None
+        if keep_logs_validator := self._validate_key_if_present(
+            key="keep_logs_after", validator=StringValidator
+        ):
+            try:
+                self._keep_logs_after = datetime_from_str(keep_logs_validator.value)
+            except Exception as exc:
+                raise self._validation_exception(f"Invalid datetime string: {str(exc)}")
+
+        self._keep_successful_logs = self._validate_key(
+            key="keep_successful_logs", validator=BoolValidator, default=True
+        )
+
+    @property
+    def logs_directory(self) -> str:
+        """
+        Required. The directory to store the logs in.
+        """
+        return self._logs_directory.value
+
+    # pylint: disable=line-too-long
+    # @property
+    # def keep_logs_after(self) -> Optional[str]:
+    #     """
+    #     Optional. Keep logs after this date, in yt-dlp datetime format.
+    #
+    #     .. code-block:: Markdown
+    #
+    #        A string in the format YYYYMMDD or
+    #        (now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s)
+    #
+    #     For example, ``today-1week`` means keep 1 week's worth of logs. By default, ytdl-sub will
+    #     keep all log files.
+    #     """
+    #     return self._keep_logs_after
+
+    # pylint: enable=line-too-long
+
+    @property
+    def keep_successful_logs(self) -> bool:
+        """
+        Optional. Whether to store logs when downloading is successful. Defaults to True.
+        """
+        return self._keep_successful_logs.value
+
+
 class ConfigOptions(StrictDictValidator):
     _required_keys = {"working_directory"}
-    _optional_keys = {"umask", "dl_aliases", "lock_directory", "ffmpeg_path", "ffprobe_path"}
+    _optional_keys = {
+        "umask",
+        "dl_aliases",
+        "persist_logs",
+        "lock_directory",
+        "ffmpeg_path",
+        "ffprobe_path",
+    }
 
     def __init__(self, name: str, value: Any):
         super().__init__(name, value)
@@ -38,6 +102,9 @@ def __init__(self, name: str, value: Any):
         self._dl_aliases = self._validate_key_if_present(
             key="dl_aliases", validator=LiteralDictValidator
         )
+        self._persist_logs = self._validate_key_if_present(
+            key="persist_logs", validator=PersistLogsValidator
+        )
         self._lock_directory = self._validate_key(
             key="lock_directory", validator=StringValidator, default=_DEFAULT_LOCK_DIRECTORY
         )
@@ -93,6 +160,13 @@ def dl_aliases(self) -> Optional[Dict[str, str]]:
             return self._dl_aliases.dict
         return {}
 
+    @property
+    def persist_logs(self) -> Optional[PersistLogsValidator]:
+        """
+        Persist logs validator. readthedocs in the validator itself!
+        """
+        return self._persist_logs
+
     @property
     def lock_directory(self) -> str:
         """

src/ytdl_sub/main.py

@@ -1,8 +1,6 @@
 import sys
 
-from ytdl_sub import __local_version__
 from ytdl_sub.cli.main_args_parser import parser
-from ytdl_sub.utils.exceptions import ValidationException
 from ytdl_sub.utils.logger import Logger
 
 
@@ -24,22 +22,11 @@ def main():
     """
     Entrypoint for ytdl-sub
     """
-    logger = Logger.get()
     try:
         _main()
         Logger.cleanup()  # Ran successfully, so we can delete the debug file
-    except ValidationException as validation_exception:
-        logger.error(validation_exception)
-        sys.exit(1)
-    except Exception:  # pylint: disable=broad-except
-        logger.exception("An uncaught error occurred:")
-        logger.error(
-            "Version %s\nPlease upload the error log file '%s' and make a Github "
-            "issue at https://github.com/jmbannon/ytdl-sub/issues with your config and "
-            "command/subscription yaml file to reproduce. Thanks for trying ytdl-sub!",
-            __local_version__,
-            Logger.debug_log_filename(),
-        )
+    except Exception as exc:  # pylint: disable=broad-except
+        Logger.log_exit_exception(exception=exc)
         sys.exit(1)
 
     sys.exit(0)