From 6e9ea638a40bd938811a9eb157d7035e036eee26 Mon Sep 17 00:00:00 2001
From: Scrub <72096833+ScrubN@users.noreply.github.com>
Date: Wed, 25 Dec 2024 23:46:16 -0500
Subject: [PATCH] Fix being able to download chats before they start/after they
 end via the CLI (#1272)

* Fix being able to download chats before they start/after they end via the CLI

* Improve chatdownload error handling
---
 TwitchDownloaderCore/ChatDownloader.cs | 31 +++++++++++++++++---------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/TwitchDownloaderCore/ChatDownloader.cs b/TwitchDownloaderCore/ChatDownloader.cs
index 0e20aaf0..1c0c639e 100644
--- a/TwitchDownloaderCore/ChatDownloader.cs
+++ b/TwitchDownloaderCore/ChatDownloader.cs
@@ -42,7 +42,7 @@ public ChatDownloader(ChatDownloadOptions chatDownloadOptions, ITaskProgress pro
                 "TwitchDownloader");
         }
 
-        private static async Task<List<Comment>> DownloadSection(Range downloadRange, string videoId, IProgress<int> progress, ChatFormat format, CancellationToken cancellationToken)
+        private static async Task<List<Comment>> DownloadSection(Range downloadRange, string videoId, IProgress<int> progress, ITaskLogger logger, ChatFormat format, CancellationToken cancellationToken)
         {
             var comments = new List<Comment>();
             int videoStart = downloadRange.Start.Value;
@@ -51,7 +51,8 @@ private static async Task<List<Comment>> DownloadSection(Range downloadRange, st
             double latestMessage = videoStart - 1;
             bool isFirst = true;
             string cursor = "";
-            int errorCount = 0;
+            double errorCount = 0;
+            double nullCount = 0;
 
             while (latestMessage < videoEnd)
             {
@@ -83,26 +84,36 @@ private static async Task<List<Comment>> DownloadSection(Range downloadRange, st
                         httpResponse.EnsureSuccessStatusCode();
                         commentResponse = await httpResponse.Content.ReadFromJsonAsync<GqlCommentResponse>(options: null, cancellationToken);
                     }
-
-                    errorCount = 0;
                 }
-                catch (HttpRequestException)
+                catch (HttpRequestException ex)
                 {
                     if (++errorCount > 10)
                     {
                         throw;
                     }
 
-                    await Task.Delay(1_000 * errorCount, cancellationToken);
+                    logger.LogVerbose($"Exception '{ex.Message}' thrown at {latestMessage}s ({cursor}) in range {downloadRange}. Current error factor: {errorCount}.");
+                    await Task.Delay((int)(1_000 * errorCount), cancellationToken);
                     continue;
                 }
 
+                // video.comments can be null for some dumb reason
                 if (commentResponse.data.video.comments?.edges is null)
                 {
-                    // video.comments can be null for some dumb reason, skip
+                    if (++nullCount > 10)
+                    {
+                        throw new Exception("Received too many null comment lists. Try reducing your download threads.");
+                    }
+
+                    logger.LogVerbose($"Received null comment list at {latestMessage}s ({cursor}) in range {downloadRange}. Current null factor: {nullCount}.");
+                    await Task.Delay((int)(100 * nullCount), cancellationToken);
                     continue;
                 }
 
+                const double BACK_OFF_FACTOR = 0.1;
+                nullCount = Math.Max(0, nullCount - BACK_OFF_FACTOR);
+                errorCount = Math.Max(0, errorCount - BACK_OFF_FACTOR);
+
                 var convertedComments = ConvertComments(commentResponse.data.video, format);
                 foreach (var comment in convertedComments)
                 {
@@ -336,8 +347,8 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF
                 chatRoot.video.description = videoInfoResponse.data.video.description?.Replace("  \n", "\n").Replace("\n\n", "\n").TrimEnd();
                 chatRoot.video.title = videoInfoResponse.data.video.title;
                 chatRoot.video.created_at = videoInfoResponse.data.video.createdAt;
-                chatRoot.video.start = downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : 0.0;
-                chatRoot.video.end = downloadOptions.TrimEnding ? downloadOptions.TrimEndingTime : videoInfoResponse.data.video.lengthSeconds;
+                chatRoot.video.start = downloadOptions.TrimBeginning ? Math.Max(0, downloadOptions.TrimBeginningTime) : 0.0;
+                chatRoot.video.end = downloadOptions.TrimEnding ? Math.Min(downloadOptions.TrimEndingTime, videoInfoResponse.data.video.lengthSeconds) : videoInfoResponse.data.video.lengthSeconds;
                 chatRoot.video.length = videoInfoResponse.data.video.lengthSeconds;
                 chatRoot.video.viewCount = videoInfoResponse.data.video.viewCount;
                 chatRoot.video.game = videoInfoResponse.data.video.game?.displayName ?? "Unknown";
@@ -437,7 +448,7 @@ private async Task<List<Comment>> DownloadComments(CancellationToken cancellatio
                 var start = videoStart + chunkSize * i;
                 var end = Math.Min(videoEnd, start + chunkSize);
                 var downloadRange = new Range(start, end);
-                downloadTasks.Add(DownloadSection(downloadRange, video.id, taskProgress, downloadOptions.DownloadFormat, cancellationToken));
+                downloadTasks.Add(DownloadSection(downloadRange, video.id, taskProgress, _progress, downloadOptions.DownloadFormat, cancellationToken));
             }
 
             await Task.WhenAll(downloadTasks);