From 46680f5b8cb2ffd1aea8a84d6e1ea5108d68ce83 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sun, 15 Oct 2023 01:37:34 -0400 Subject: [PATCH 1/7] Fix incorrect percentages in chat updater --- TwitchDownloaderCore/ChatUpdater.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/TwitchDownloaderCore/ChatUpdater.cs b/TwitchDownloaderCore/ChatUpdater.cs index 2fb1b315..cb998e46 100644 --- a/TwitchDownloaderCore/ChatUpdater.cs +++ b/TwitchDownloaderCore/ChatUpdater.cs @@ -60,7 +60,7 @@ public async Task UpdateAsync(IProgress progress, CancellationTo // Finally save the output to file! progress.Report(new ProgressReport(ReportType.NewLineStatus, $"Writing Output File [{++currentStep}/{totalSteps}]")); - progress.Report(new ProgressReport(totalSteps / currentStep)); + progress.Report(new ProgressReport(currentStep * 100 / totalSteps)); switch (_updateOptions.OutputFormat) { @@ -81,9 +81,7 @@ public async Task UpdateAsync(IProgress progress, CancellationTo private async Task UpdateChatCrop(int totalSteps, int currentStep, IProgress progress, CancellationToken cancellationToken) { progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Updating Chat Crop [{currentStep}/{totalSteps}]")); - progress.Report(new ProgressReport(totalSteps / currentStep)); - - chatRoot.video ??= new Video(); + progress.Report(new ProgressReport(currentStep * 100 / totalSteps)); bool cropTaskVodExpired = false; var cropTaskProgress = new Progress(report => @@ -145,7 +143,7 @@ private async Task UpdateChatCrop(int totalSteps, int currentStep, IProgress progress, CancellationToken cancellationToken) { progress.Report(new ProgressReport(ReportType.NewLineStatus, $"Updating Embeds [{currentStep}/{totalSteps}]")); - progress.Report(new ProgressReport(totalSteps / currentStep)); + progress.Report(new ProgressReport(currentStep * 100 / totalSteps)); chatRoot.embeddedData ??= new EmbeddedData(); From 81579a60301cc4ad7463007cb65d0f63c068e1e6 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sun, 15 Oct 2023 01:37:58 -0400 Subject: [PATCH 2/7] Make crop lock object not static --- TwitchDownloaderCore/ChatUpdater.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/TwitchDownloaderCore/ChatUpdater.cs b/TwitchDownloaderCore/ChatUpdater.cs index cb998e46..81a3c886 100644 --- a/TwitchDownloaderCore/ChatUpdater.cs +++ b/TwitchDownloaderCore/ChatUpdater.cs @@ -14,6 +14,7 @@ namespace TwitchDownloaderCore public sealed class ChatUpdater { public ChatRoot chatRoot { get; internal set; } = new(); + private readonly object _cropChatRootLock = new(); private readonly ChatUpdateOptions _updateOptions; @@ -25,11 +26,6 @@ public ChatUpdater(ChatUpdateOptions updateOptions) "TwitchDownloader"); } - private static class SharedObjects - { - internal static object CropChatRootLock = new(); - } - public async Task UpdateAsync(IProgress progress, CancellationToken cancellationToken) { chatRoot.FileInfo = new() { Version = ChatRootVersion.CurrentVersion, CreatedAt = chatRoot.FileInfo.CreatedAt, UpdatedAt = DateTime.Now }; @@ -323,7 +319,7 @@ private async Task ChatEndingCropTask(IProgress progress, Cancel } } - lock (SharedObjects.CropChatRootLock) + lock (_cropChatRootLock) { foreach (var comment in chatRoot.comments) { From f42c2d205daea5f9f20ef0272e52c46b818f629f Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sun, 15 Oct 2023 01:52:53 -0400 Subject: [PATCH 3/7] Update video info if possible when updating chats --- TwitchDownloaderCore/ChatUpdater.cs | 78 ++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/TwitchDownloaderCore/ChatUpdater.cs b/TwitchDownloaderCore/ChatUpdater.cs index 81a3c886..88a97efc 100644 --- a/TwitchDownloaderCore/ChatUpdater.cs +++ b/TwitchDownloaderCore/ChatUpdater.cs @@ -8,6 +8,7 @@ using TwitchDownloaderCore.Options; using TwitchDownloaderCore.Tools; using TwitchDownloaderCore.TwitchObjects; +using TwitchDownloaderCore.TwitchObjects.Gql; namespace TwitchDownloaderCore { @@ -36,10 +37,13 @@ public async Task UpdateAsync(IProgress progress, CancellationTo // Dynamic step count setup int currentStep = 0; - int totalSteps = 1; + int totalSteps = 2; if (_updateOptions.CropBeginning || _updateOptions.CropEnding) totalSteps++; if (_updateOptions.EmbedMissing || _updateOptions.ReplaceEmbeds) totalSteps++; + currentStep++; + await UpdateVideoInfo(totalSteps, currentStep, progress, cancellationToken); + // If we are editing the chat crop if (_updateOptions.CropBeginning || _updateOptions.CropEnding) { @@ -74,6 +78,76 @@ public async Task UpdateAsync(IProgress progress, CancellationTo } } + private async Task UpdateVideoInfo(int totalSteps, int currentStep, IProgress progress, CancellationToken cancellationToken) + { + progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Updating Video Info [{currentStep}/{totalSteps}]")); + progress.Report(new ProgressReport(currentStep * 100 / totalSteps)); + + if (chatRoot.video.id.All(char.IsDigit)) + { + var videoId = int.Parse(chatRoot.video.id); + VideoInfo videoInfo = null; + try + { + videoInfo = (await TwitchHelper.GetVideoInfo(videoId)).data.video; + } + catch { /* Eat the exception */ } + + if (videoInfo is null) + { + progress.Report(new ProgressReport(ReportType.SameLineStatus, "Unable to fetch video info, deleted/expired VOD possibly?")); + return; + } + + chatRoot.video.title = videoInfo.title; + chatRoot.video.description = videoInfo.description; + chatRoot.video.created_at = videoInfo.createdAt; + chatRoot.video.length = videoInfo.lengthSeconds; + chatRoot.video.viewCount = videoInfo.viewCount; + chatRoot.video.game = videoInfo.game.displayName; + + var chaptersInfo = (await TwitchHelper.GetVideoChapters(videoId)).data.video.moments.edges; + foreach (var responseChapter in chaptersInfo) + { + chatRoot.video.chapters.Add(new VideoChapter + { + id = responseChapter.node.id, + startMilliseconds = responseChapter.node.positionMilliseconds, + lengthMilliseconds = responseChapter.node.durationMilliseconds, + _type = responseChapter.node._type, + description = responseChapter.node.description, + subDescription = responseChapter.node.subDescription, + thumbnailUrl = responseChapter.node.thumbnailURL, + gameId = responseChapter.node.details.game?.id, + gameDisplayName = responseChapter.node.details.game?.displayName, + gameBoxArtUrl = responseChapter.node.details.game?.boxArtURL + }); + } + } + else + { + var clipId = chatRoot.video.id; + Clip clipInfo = null; + try + { + clipInfo = (await TwitchHelper.GetClipInfo(clipId)).data.clip; + } + catch { /* Eat the exception */ } + + if (clipInfo is null) + { + progress.Report(new ProgressReport(ReportType.SameLineStatus, "Unable to fetch clip info, deleted possibly?")); + return; + } + + chatRoot.video.title = clipInfo.title; + chatRoot.video.created_at = clipInfo.createdAt; + chatRoot.video.length = clipInfo.durationSeconds; + chatRoot.video.viewCount = clipInfo.viewCount; + chatRoot.video.game = clipInfo.game.displayName; + } + } + private async Task UpdateChatCrop(int totalSteps, int currentStep, IProgress progress, CancellationToken cancellationToken) { progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Updating Chat Crop [{currentStep}/{totalSteps}]")); @@ -82,7 +156,7 @@ private async Task UpdateChatCrop(int totalSteps, int currentStep, IProgress(report => { - if (((string)report.Data).ToLower().Contains("vod is expired")) + if (((string)report.Data).Contains("vod is expired", StringComparison.OrdinalIgnoreCase)) { // If the user is moving both crops in one command, we only want to propagate a 'vod expired/id corrupt' report once if (cropTaskVodExpired) From f2daa6779f8d6687f3fbae88e788158cdc87a7d1 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sun, 15 Oct 2023 02:14:55 -0400 Subject: [PATCH 4/7] Compress chat crop updater temp files with gzip --- TwitchDownloaderCore/ChatUpdater.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/TwitchDownloaderCore/ChatUpdater.cs b/TwitchDownloaderCore/ChatUpdater.cs index 88a97efc..ef76f3b3 100644 --- a/TwitchDownloaderCore/ChatUpdater.cs +++ b/TwitchDownloaderCore/ChatUpdater.cs @@ -413,6 +413,7 @@ private ChatDownloadOptions GetCropDownloadOptions(string videoId, string tempFi { Id = videoId, DownloadFormat = ChatFormat.Json, // json is required to parse as a new chatroot object + Compression = ChatCompression.Gzip, Filename = tempFile, CropBeginning = true, CropBeginningTime = sectionStart, From c69f2d78105b702e8e52f85f02fff4551ce45880 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sun, 15 Oct 2023 03:40:08 -0400 Subject: [PATCH 5/7] Fix ArgumentOutOfRangeException when loading information from chat files with less than 2 comments --- TwitchDownloaderWPF/PageChatUpdate.xaml.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/TwitchDownloaderWPF/PageChatUpdate.xaml.cs b/TwitchDownloaderWPF/PageChatUpdate.xaml.cs index 0acbd49e..308f5ea5 100644 --- a/TwitchDownloaderWPF/PageChatUpdate.xaml.cs +++ b/TwitchDownloaderWPF/PageChatUpdate.xaml.cs @@ -67,7 +67,10 @@ private async void btnBrowse_Click(object sender, RoutedEventArgs e) try { ChatJsonInfo = await ChatJson.DeserializeAsync(InputFile, true, false, CancellationToken.None); - ChatJsonInfo.comments.RemoveRange(1, ChatJsonInfo.comments.Count - 2); + if (ChatJsonInfo.comments.Count > 2) + { + ChatJsonInfo.comments.RemoveRange(1, ChatJsonInfo.comments.Count - 2); + } GC.Collect(); } catch (Exception ex) From 16302c742ac6785a76aa873c00de9bd32684bf55 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Thu, 19 Oct 2023 00:35:02 -0400 Subject: [PATCH 6/7] Add functionality to deserialize only the first and last comments --- TwitchDownloaderCore/Chat/ChatJson.cs | 6 ++-- TwitchDownloaderCore/ChatRenderer.cs | 2 +- TwitchDownloaderCore/ChatUpdater.cs | 4 +-- .../Tools/JsonElementExtensions.cs | 32 +++++++++++++++++++ TwitchDownloaderWPF/PageChatUpdate.xaml.cs | 6 +--- 5 files changed, 40 insertions(+), 10 deletions(-) create mode 100644 TwitchDownloaderCore/Tools/JsonElementExtensions.cs diff --git a/TwitchDownloaderCore/Chat/ChatJson.cs b/TwitchDownloaderCore/Chat/ChatJson.cs index 6e04b41f..05946e36 100644 --- a/TwitchDownloaderCore/Chat/ChatJson.cs +++ b/TwitchDownloaderCore/Chat/ChatJson.cs @@ -28,7 +28,7 @@ public static class ChatJson /// A representation the deserialized chat json file. /// The file does not exist. /// The file is not a valid chat format. - public static async Task DeserializeAsync(string filePath, bool getComments = true, bool getEmbeds = true, CancellationToken cancellationToken = new()) + public static async Task DeserializeAsync(string filePath, bool getComments = true, bool onlyFirstAndLastComments = false, bool getEmbeds = true, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(filePath, nameof(filePath)); @@ -92,7 +92,9 @@ public static class ChatJson { if (jsonDocument.RootElement.TryGetProperty("comments", out JsonElement commentsElement)) { - returnChatRoot.comments = commentsElement.Deserialize>(options: _jsonSerializerOptions); + returnChatRoot.comments = onlyFirstAndLastComments + ? commentsElement.DeserializeFirstAndLastFromList(options: _jsonSerializerOptions) + : commentsElement.Deserialize>(options: _jsonSerializerOptions); } } diff --git a/TwitchDownloaderCore/ChatRenderer.cs b/TwitchDownloaderCore/ChatRenderer.cs index 4059f8b4..c735dad8 100644 --- a/TwitchDownloaderCore/ChatRenderer.cs +++ b/TwitchDownloaderCore/ChatRenderer.cs @@ -1654,7 +1654,7 @@ private static bool IsRightToLeft(ReadOnlySpan message) public async Task ParseJsonAsync(CancellationToken cancellationToken = new()) { - chatRoot = await ChatJson.DeserializeAsync(renderOptions.InputFile, true, true, cancellationToken); + chatRoot = await ChatJson.DeserializeAsync(renderOptions.InputFile, true, false, true, cancellationToken); return chatRoot; } diff --git a/TwitchDownloaderCore/ChatUpdater.cs b/TwitchDownloaderCore/ChatUpdater.cs index ef76f3b3..6c9ac181 100644 --- a/TwitchDownloaderCore/ChatUpdater.cs +++ b/TwitchDownloaderCore/ChatUpdater.cs @@ -381,7 +381,7 @@ private async Task ChatEndingCropTask(IProgress progress, Cancel ChatDownloader chatDownloader = new ChatDownloader(downloadOptions); await chatDownloader.DownloadAsync(new Progress(), cancellationToken); - ChatRoot newChatRoot = await ChatJson.DeserializeAsync(inputFile, getComments: true, getEmbeds: false, cancellationToken); + ChatRoot newChatRoot = await ChatJson.DeserializeAsync(inputFile, getComments: true, onlyFirstAndLastComments: false, getEmbeds: false, cancellationToken); // Append the new comment section SortedSet commentsSet = new SortedSet(new SortedCommentComparer()); @@ -430,7 +430,7 @@ private ChatDownloadOptions GetCropDownloadOptions(string videoId, string tempFi public async Task ParseJsonAsync(CancellationToken cancellationToken = new()) { - chatRoot = await ChatJson.DeserializeAsync(_updateOptions.InputFile, true, true, cancellationToken); + chatRoot = await ChatJson.DeserializeAsync(_updateOptions.InputFile, true, false, true, cancellationToken); return chatRoot; } } diff --git a/TwitchDownloaderCore/Tools/JsonElementExtensions.cs b/TwitchDownloaderCore/Tools/JsonElementExtensions.cs new file mode 100644 index 00000000..24f0e82e --- /dev/null +++ b/TwitchDownloaderCore/Tools/JsonElementExtensions.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; +using System.Text.Json; + +namespace TwitchDownloaderCore.Tools +{ + public static class JsonElementExtensions + { + public static List DeserializeFirstAndLastFromList(this JsonElement arrayElement, JsonSerializerOptions options = null) + { + // It's not the prettiest, but for arrays with thousands of objects it can save whole seconds and prevent tons of fragmented memory + var list = new List(2); + JsonElement lastElement = default; + foreach (var element in arrayElement.EnumerateArray()) + { + if (list.Count == 0) + { + list.Add(element.Deserialize(options: options)); + continue; + } + + lastElement = element; + } + + if (lastElement.ValueKind != JsonValueKind.Undefined) + { + list.Add(lastElement.Deserialize(options: options)); + } + + return list; + } + } +} \ No newline at end of file diff --git a/TwitchDownloaderWPF/PageChatUpdate.xaml.cs b/TwitchDownloaderWPF/PageChatUpdate.xaml.cs index 308f5ea5..706ac3f3 100644 --- a/TwitchDownloaderWPF/PageChatUpdate.xaml.cs +++ b/TwitchDownloaderWPF/PageChatUpdate.xaml.cs @@ -66,11 +66,7 @@ private async void btnBrowse_Click(object sender, RoutedEventArgs e) try { - ChatJsonInfo = await ChatJson.DeserializeAsync(InputFile, true, false, CancellationToken.None); - if (ChatJsonInfo.comments.Count > 2) - { - ChatJsonInfo.comments.RemoveRange(1, ChatJsonInfo.comments.Count - 2); - } + ChatJsonInfo = await ChatJson.DeserializeAsync(InputFile, true, true, false, CancellationToken.None); GC.Collect(); } catch (Exception ex) From 446923e85a822353c8f88aa08674b5c9b9893acc Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Fri, 27 Oct 2023 23:30:14 -0400 Subject: [PATCH 7/7] Fix chapter updating --- TwitchDownloaderCore/ChatUpdater.cs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/TwitchDownloaderCore/ChatUpdater.cs b/TwitchDownloaderCore/ChatUpdater.cs index 6c9ac181..5e1dd01f 100644 --- a/TwitchDownloaderCore/ChatUpdater.cs +++ b/TwitchDownloaderCore/ChatUpdater.cs @@ -106,7 +106,7 @@ private async Task UpdateVideoInfo(int totalSteps, int currentStep, IProgress