Many chat updater fixes

TwitchDownloaderCore/Chat/ChatJson.cs

@@ -29,7 +29,7 @@ public static class ChatJson
         /// <returns>A <see cref="ChatRoot"/> representation the deserialized chat json file.</returns>
         /// <exception cref="IOException">The file does not exist.</exception>
         /// <exception cref="NotSupportedException">The file is not a valid chat format.</exception>
-        public static async Task<ChatRoot> DeserializeAsync(string filePath, bool getComments = true, bool getEmbeds = true, CancellationToken cancellationToken = new())
+        public static async Task<ChatRoot> DeserializeAsync(string filePath, bool getComments = true, bool onlyFirstAndLastComments = false, bool getEmbeds = true, CancellationToken cancellationToken = default)
         {
             ArgumentNullException.ThrowIfNull(filePath, nameof(filePath));
 
@@ -82,7 +82,9 @@ public static class ChatJson
             {
                 if (jsonDocument.RootElement.TryGetProperty("comments", out JsonElement commentsElement))
                 {
-                    returnChatRoot.comments = commentsElement.Deserialize<List<Comment>>(options: _jsonSerializerOptions);
+                    returnChatRoot.comments = onlyFirstAndLastComments
+                        ? commentsElement.DeserializeFirstAndLastFromList<Comment>(options: _jsonSerializerOptions)
+                        : commentsElement.Deserialize<List<Comment>>(options: _jsonSerializerOptions);
                 }
             }
 

TwitchDownloaderCore/ChatRenderer.cs

@@ -1694,7 +1694,7 @@ private static bool IsRightToLeft(ReadOnlySpan<char> message)
 
         public async Task<ChatRoot> ParseJsonAsync(CancellationToken cancellationToken = new())
         {
-            chatRoot = await ChatJson.DeserializeAsync(renderOptions.InputFile, true, true, cancellationToken);
+            chatRoot = await ChatJson.DeserializeAsync(renderOptions.InputFile, true, false, true, cancellationToken);
             return chatRoot;
         }
 

TwitchDownloaderCore/ChatUpdater.cs

@@ -8,12 +8,14 @@
 using TwitchDownloaderCore.Options;
 using TwitchDownloaderCore.Tools;
 using TwitchDownloaderCore.TwitchObjects;
+using TwitchDownloaderCore.TwitchObjects.Gql;
 
 namespace TwitchDownloaderCore
 {
     public sealed class ChatUpdater
     {
         public ChatRoot chatRoot { get; internal set; } = new();
+        private readonly object _cropChatRootLock = new();
 
         private readonly ChatUpdateOptions _updateOptions;
 
@@ -25,11 +27,6 @@ public ChatUpdater(ChatUpdateOptions updateOptions)
                 "TwitchDownloader");
         }
 
-        private static class SharedObjects
-        {
-            internal static object CropChatRootLock = new();
-        }
-
         public async Task UpdateAsync(IProgress<ProgressReport> progress, CancellationToken cancellationToken)
         {
             chatRoot.FileInfo = new() { Version = ChatRootVersion.CurrentVersion, CreatedAt = chatRoot.FileInfo.CreatedAt, UpdatedAt = DateTime.Now };
@@ -40,10 +37,13 @@ public async Task UpdateAsync(IProgress<ProgressReport> progress, CancellationTo
 
             // Dynamic step count setup
             int currentStep = 0;
-            int totalSteps = 1;
+            int totalSteps = 2;
             if (_updateOptions.CropBeginning || _updateOptions.CropEnding) totalSteps++;
             if (_updateOptions.EmbedMissing || _updateOptions.ReplaceEmbeds) totalSteps++;
 
+            currentStep++;
+            await UpdateVideoInfo(totalSteps, currentStep, progress, cancellationToken);
+
             // If we are editing the chat crop
             if (_updateOptions.CropBeginning || _updateOptions.CropEnding)
             {
@@ -60,7 +60,7 @@ public async Task UpdateAsync(IProgress<ProgressReport> progress, CancellationTo
 
             // Finally save the output to file!
             progress.Report(new ProgressReport(ReportType.NewLineStatus, $"Writing Output File [{++currentStep}/{totalSteps}]"));
-            progress.Report(new ProgressReport(totalSteps / currentStep));
+            progress.Report(new ProgressReport(currentStep * 100 / totalSteps));
 
             switch (_updateOptions.OutputFormat)
             {
@@ -78,17 +78,100 @@ public async Task UpdateAsync(IProgress<ProgressReport> progress, CancellationTo
             }
         }
 
+        private async Task UpdateVideoInfo(int totalSteps, int currentStep, IProgress<ProgressReport> progress, CancellationToken cancellationToken)
+        {
+            progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Updating Video Info [{currentStep}/{totalSteps}]"));
+            progress.Report(new ProgressReport(currentStep * 100 / totalSteps));
+
+            if (chatRoot.video.id.All(char.IsDigit))
+            {
+                var videoId = int.Parse(chatRoot.video.id);
+                VideoInfo videoInfo = null;
+                try
+                {
+                    videoInfo = (await TwitchHelper.GetVideoInfo(videoId)).data.video;
+                }
+                catch { /* Eat the exception */ }
+
+                if (videoInfo is null)
+                {
+                    progress.Report(new ProgressReport(ReportType.SameLineStatus, "Unable to fetch video info, deleted/expired VOD possibly?"));
+                    return;
+                }
+
+                chatRoot.video.title = videoInfo.title;
+                chatRoot.video.description = videoInfo.description;
+                chatRoot.video.created_at = videoInfo.createdAt;
+                chatRoot.video.length = videoInfo.lengthSeconds;
+                chatRoot.video.viewCount = videoInfo.viewCount;
+                chatRoot.video.game = videoInfo.game.displayName;
+
+                var chaptersInfo = (await TwitchHelper.GetOrGenerateVideoChapters(videoId, videoInfo)).data.video.moments.edges;
+                foreach (var responseChapter in chaptersInfo)
+                {
+                    chatRoot.video.chapters.Add(new VideoChapter
+                    {
+                        id = responseChapter.node.id,
+                        startMilliseconds = responseChapter.node.positionMilliseconds,
+                        lengthMilliseconds = responseChapter.node.durationMilliseconds,
+                        _type = responseChapter.node._type,
+                        description = responseChapter.node.description,
+                        subDescription = responseChapter.node.subDescription,
+                        thumbnailUrl = responseChapter.node.thumbnailURL,
+                        gameId = responseChapter.node.details.game?.id,
+                        gameDisplayName = responseChapter.node.details.game?.displayName,
+                        gameBoxArtUrl = responseChapter.node.details.game?.boxArtURL
+                    });
+                }
+            }
+            else
+            {
+                var clipId = chatRoot.video.id;
+                Clip clipInfo = null;
+                try
+                {
+                    clipInfo = (await TwitchHelper.GetClipInfo(clipId)).data.clip;
+                }
+                catch { /* Eat the exception */ }
+
+                if (clipInfo is null)
+                {
+                    progress.Report(new ProgressReport(ReportType.SameLineStatus, "Unable to fetch clip info, deleted possibly?"));
+                    return;
+                }
+
+                chatRoot.video.title = clipInfo.title;
+                chatRoot.video.created_at = clipInfo.createdAt;
+                chatRoot.video.length = clipInfo.durationSeconds;
+                chatRoot.video.viewCount = clipInfo.viewCount;
+                chatRoot.video.game = clipInfo.game.displayName;
+
+                var clipChapter = TwitchHelper.GenerateClipChapter(clipInfo);
+                chatRoot.video.chapters.Add(new VideoChapter
+                {
+                    id = clipChapter.node.id,
+                    startMilliseconds = clipChapter.node.positionMilliseconds,
+                    lengthMilliseconds = clipChapter.node.durationMilliseconds,
+                    _type = clipChapter.node._type,
+                    description = clipChapter.node.description,
+                    subDescription = clipChapter.node.subDescription,
+                    thumbnailUrl = clipChapter.node.thumbnailURL,
+                    gameId = clipChapter.node.details.game?.id,
+                    gameDisplayName = clipChapter.node.details.game?.displayName,
+                    gameBoxArtUrl = clipChapter.node.details.game?.boxArtURL
+                });
+            }
+        }
+
         private async Task UpdateChatCrop(int totalSteps, int currentStep, IProgress<ProgressReport> progress, CancellationToken cancellationToken)
         {
             progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Updating Chat Crop [{currentStep}/{totalSteps}]"));
-            progress.Report(new ProgressReport(totalSteps / currentStep));
-
-            chatRoot.video ??= new Video();
+            progress.Report(new ProgressReport(currentStep * 100 / totalSteps));
 
             bool cropTaskVodExpired = false;
             var cropTaskProgress = new Progress<ProgressReport>(report =>
             {
-                if (((string)report.Data).ToLower().Contains("vod is expired"))
+                if (((string)report.Data).Contains("vod is expired", StringComparison.OrdinalIgnoreCase))
                 {
                     // If the user is moving both crops in one command, we only want to propagate a 'vod expired/id corrupt' report once
                     if (cropTaskVodExpired)
@@ -145,7 +228,7 @@ private async Task UpdateChatCrop(int totalSteps, int currentStep, IProgress<Pro
         private async Task UpdateEmbeds(int currentStep, int totalSteps, IProgress<ProgressReport> progress, CancellationToken cancellationToken)
         {
             progress.Report(new ProgressReport(ReportType.NewLineStatus, $"Updating Embeds [{currentStep}/{totalSteps}]"));
-            progress.Report(new ProgressReport(totalSteps / currentStep));
+            progress.Report(new ProgressReport(currentStep * 100 / totalSteps));
 
             chatRoot.embeddedData ??= new EmbeddedData();
 
@@ -313,7 +396,7 @@ private async Task ChatEndingCropTask(IProgress<ProgressReport> progress, Cancel
             ChatDownloader chatDownloader = new ChatDownloader(downloadOptions);
             await chatDownloader.DownloadAsync(new Progress<ProgressReport>(), cancellationToken);
 
-            ChatRoot newChatRoot = await ChatJson.DeserializeAsync(inputFile, getComments: true, getEmbeds: false, cancellationToken);
+            ChatRoot newChatRoot = await ChatJson.DeserializeAsync(inputFile, getComments: true, onlyFirstAndLastComments: false, getEmbeds: false, cancellationToken);
 
             // Append the new comment section
             SortedSet<Comment> commentsSet = new SortedSet<Comment>(new SortedCommentComparer());
@@ -325,7 +408,7 @@ private async Task ChatEndingCropTask(IProgress<ProgressReport> progress, Cancel
                 }
             }
 
-            lock (SharedObjects.CropChatRootLock)
+            lock (_cropChatRootLock)
             {
                 foreach (var comment in chatRoot.comments)
                 {
@@ -345,6 +428,7 @@ private ChatDownloadOptions GetCropDownloadOptions(string videoId, string tempFi
             {
                 Id = videoId,
                 DownloadFormat = ChatFormat.Json, // json is required to parse as a new chatroot object
+                Compression = ChatCompression.Gzip,
                 Filename = tempFile,
                 CropBeginning = true,
                 CropBeginningTime = sectionStart,
@@ -361,7 +445,7 @@ private ChatDownloadOptions GetCropDownloadOptions(string videoId, string tempFi
 
         public async Task<ChatRoot> ParseJsonAsync(CancellationToken cancellationToken = new())
         {
-            chatRoot = await ChatJson.DeserializeAsync(_updateOptions.InputFile, true, true, cancellationToken);
+            chatRoot = await ChatJson.DeserializeAsync(_updateOptions.InputFile, true, false, true, cancellationToken);
             return chatRoot;
         }
     }

TwitchDownloaderCore/Tools/JsonElementExtensions.cs

@@ -0,0 +1,32 @@
+using System.Collections.Generic;
+using System.Text.Json;
+
+namespace TwitchDownloaderCore.Tools
+{
+    public static class JsonElementExtensions
+    {
+        public static List<T> DeserializeFirstAndLastFromList<T>(this JsonElement arrayElement, JsonSerializerOptions options = null)
+        {
+            // It's not the prettiest, but for arrays with thousands of objects it can save whole seconds and prevent tons of fragmented memory
+            var list = new List<T>(2);
+            JsonElement lastElement = default;
+            foreach (var element in arrayElement.EnumerateArray())
+            {
+                if (list.Count == 0)
+                {
+                    list.Add(element.Deserialize<T>(options: options));
+                    continue;
+                }
+
+                lastElement = element;
+            }
+
+            if (lastElement.ValueKind != JsonValueKind.Undefined)
+            {
+                list.Add(lastElement.Deserialize<T>(options: options));
+            }
+
+            return list;
+        }
+    }
+}

TwitchDownloaderWPF/PageChatUpdate.xaml.cs

@@ -66,8 +66,7 @@ private async void btnBrowse_Click(object sender, RoutedEventArgs e)
 
             try
             {
-                ChatJsonInfo = await ChatJson.DeserializeAsync(InputFile, true, false, CancellationToken.None);
-                ChatJsonInfo.comments.RemoveRange(1, ChatJsonInfo.comments.Count - 2);
+                ChatJsonInfo = await ChatJson.DeserializeAsync(InputFile, true, true, false, CancellationToken.None);
                 GC.Collect();
             }
             catch (Exception ex)