From 03a461b47e88429972ce2dd87d1a7f03b6ba5202 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sat, 10 Aug 2024 22:09:50 -0400 Subject: [PATCH 1/4] Separate FfmpegMetadata.SerializeAsync into dedicated VideoInfo and Clip methods --- TwitchDownloaderCore/ClipDownloader.cs | 3 +-- TwitchDownloaderCore/Tools/FfmpegMetadata.cs | 20 ++++++++++++++++---- TwitchDownloaderCore/VideoDownloader.cs | 3 +-- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/TwitchDownloaderCore/ClipDownloader.cs b/TwitchDownloaderCore/ClipDownloader.cs index 1a85ccbf..7068d4c0 100644 --- a/TwitchDownloaderCore/ClipDownloader.cs +++ b/TwitchDownloaderCore/ClipDownloader.cs @@ -224,8 +224,7 @@ private async Task EncodeClipWithMetadata(string inputFile, string destinationFi Process process = null; try { - await FfmpegMetadata.SerializeAsync(metadataFile, clipMetadata.broadcaster?.displayName, downloadOptions.Id, clipMetadata.title, clipMetadata.createdAt, clipMetadata.viewCount, - videoMomentEdges: new[] { clipChapter }, cancellationToken: cancellationToken); + await FfmpegMetadata.SerializeAsync(metadataFile, downloadOptions.Id, clipMetadata, new[] { clipChapter }, cancellationToken); process = new Process { diff --git a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs index 12017f75..b07a3f44 100644 --- a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs +++ b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs @@ -13,19 +13,31 @@ public static class FfmpegMetadata { private const string LINE_FEED = "\u000A"; - public static async Task SerializeAsync(string filePath, string streamerName, string videoId, string videoTitle, DateTime videoCreation, int viewCount, string videoDescription = null, - TimeSpan startOffset = default, TimeSpan videoLength = default, IEnumerable videoMomentEdges = null, CancellationToken cancellationToken = default) + public static async Task SerializeAsync(string filePath, string videoId, VideoInfo videoInfo, TimeSpan startOffset, TimeSpan videoLength, IEnumerable videoMomentEdges, CancellationToken cancellationToken = default) { await using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED }; - await SerializeGlobalMetadata(sw, streamerName, videoId, videoTitle, videoCreation, viewCount, videoDescription); + var description = videoInfo.description?.Replace(" \n", "\n").Replace("\n\n", "\n").TrimEnd(); + await SerializeGlobalMetadata(sw, videoInfo.owner.displayName, videoId, videoInfo.title, videoInfo.createdAt, videoInfo.viewCount, description); await fs.FlushAsync(cancellationToken); await SerializeChapters(sw, videoMomentEdges, startOffset, videoLength); await fs.FlushAsync(cancellationToken); } + public static async Task SerializeAsync(string filePath, string videoId, Clip clip, IEnumerable videoMomentEdges, CancellationToken cancellationToken = default) + { + await using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); + await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED }; + + await SerializeGlobalMetadata(sw, clip.broadcaster.displayName, videoId, clip.title, clip.createdAt, clip.viewCount, null); + await fs.FlushAsync(cancellationToken); + + await SerializeChapters(sw, videoMomentEdges); + await fs.FlushAsync(cancellationToken); + } + private static async Task SerializeGlobalMetadata(StreamWriter sw, string streamerName, string videoId, string videoTitle, DateTime videoCreation, int viewCount, string videoDescription) { await sw.WriteLineAsync(";FFMETADATA1"); @@ -44,7 +56,7 @@ private static async Task SerializeGlobalMetadata(StreamWriter sw, string stream await sw.WriteLineAsync(@$"Views: {viewCount}"); } - private static async Task SerializeChapters(StreamWriter sw, IEnumerable videoMomentEdges, TimeSpan startOffset, TimeSpan videoLength) + private static async Task SerializeChapters(StreamWriter sw, IEnumerable videoMomentEdges, TimeSpan startOffset = default, TimeSpan videoLength = default) { if (videoMomentEdges is null) { diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index fe2ac551..31877a77 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -105,8 +105,7 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF _progress.SetTemplateStatus("Finalizing Video {0}% [4/4]", 0); string metadataPath = Path.Combine(downloadFolder, "metadata.txt"); - await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, downloadOptions.Id.ToString(), videoInfo.title, videoInfo.createdAt, videoInfo.viewCount, - videoInfo.description?.Replace(" \n", "\n").Replace("\n\n", "\n").TrimEnd(), downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : TimeSpan.Zero, + await FfmpegMetadata.SerializeAsync(metadataPath, downloadOptions.Id.ToString(), videoInfo, downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : TimeSpan.Zero, videoLength, videoChapterResponse.data.video.moments.edges, cancellationToken); var concatListPath = Path.Combine(downloadFolder, "concat.txt"); From a195705e43da55f4ac54b0ff0e3f3ac762a0123e Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sat, 10 Aug 2024 22:20:14 -0400 Subject: [PATCH 2/4] Embed more metadata in videos and clips --- TwitchDownloaderCore/Tools/FfmpegMetadata.cs | 55 ++++++++++++++++---- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs index b07a3f44..8e92cd48 100644 --- a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs +++ b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.IO; +using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -18,8 +20,9 @@ public static async Task SerializeAsync(string filePath, string videoId, VideoIn await using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED }; + var streamer = GetUserName(videoInfo.owner.displayName, videoInfo.owner.login); var description = videoInfo.description?.Replace(" \n", "\n").Replace("\n\n", "\n").TrimEnd(); - await SerializeGlobalMetadata(sw, videoInfo.owner.displayName, videoId, videoInfo.title, videoInfo.createdAt, videoInfo.viewCount, description); + await SerializeGlobalMetadata(sw, streamer, videoId, videoInfo.title, videoInfo.createdAt, videoInfo.viewCount, description, videoInfo.game?.displayName); await fs.FlushAsync(cancellationToken); await SerializeChapters(sw, videoMomentEdges, startOffset, videoLength); @@ -31,28 +34,37 @@ public static async Task SerializeAsync(string filePath, string videoId, Clip cl await using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED }; - await SerializeGlobalMetadata(sw, clip.broadcaster.displayName, videoId, clip.title, clip.createdAt, clip.viewCount, null); + var streamer = GetUserName(clip.broadcaster.displayName, clip.broadcaster.login); + var clipper = GetUserName(clip.curator.displayName, clip.curator.login); + await SerializeGlobalMetadata(sw, streamer, videoId, clip.title, clip.createdAt, clip.viewCount, game: clip.game?.displayName, clipper: clipper); await fs.FlushAsync(cancellationToken); await SerializeChapters(sw, videoMomentEdges); await fs.FlushAsync(cancellationToken); } - private static async Task SerializeGlobalMetadata(StreamWriter sw, string streamerName, string videoId, string videoTitle, DateTime videoCreation, int viewCount, string videoDescription) + private static async Task SerializeGlobalMetadata(StreamWriter sw, [AllowNull] string streamer, string id, string title, DateTime createdAt, int viewCount, [AllowNull] string description = null, [AllowNull] string game = null, + [AllowNull] string clipper = null) { + // ReSharper disable once StringLiteralTypo await sw.WriteLineAsync(";FFMETADATA1"); - await sw.WriteLineAsync($"title={SanitizeKeyValue(videoTitle)} ({SanitizeKeyValue(videoId)})"); - if (!string.IsNullOrWhiteSpace(streamerName)) - await sw.WriteLineAsync($"artist={SanitizeKeyValue(streamerName)}"); - await sw.WriteLineAsync($"date={videoCreation:yyyy}"); // The 'date' key becomes 'year' in most formats + await sw.WriteLineAsync($"title={SanitizeKeyValue(title)} ({SanitizeKeyValue(id)})"); + if (!string.IsNullOrWhiteSpace(streamer)) + await sw.WriteLineAsync($"artist={SanitizeKeyValue(streamer)}"); + await sw.WriteLineAsync($"date={createdAt:yyyy}"); // The 'date' key becomes 'year' in most formats + if (!string.IsNullOrWhiteSpace(game)) + await sw.WriteLineAsync($"genre={game}"); await sw.WriteAsync(@"comment="); - if (!string.IsNullOrWhiteSpace(videoDescription)) + if (!string.IsNullOrWhiteSpace(description)) { - await sw.WriteLineAsync(@$"{SanitizeKeyValue(videoDescription.TrimEnd())}\"); + // We could use the 'description' key, but so few media players support mp4 descriptions that users would probably think it was missing + await sw.WriteLineAsync(@$"{SanitizeKeyValue(description.TrimEnd())}\"); await sw.WriteLineAsync(@"------------------------\"); } - await sw.WriteLineAsync(@$"Originally aired: {SanitizeKeyValue(videoCreation.ToString("u"))}\"); - await sw.WriteLineAsync(@$"Video id: {SanitizeKeyValue(videoId)}\"); + if (!string.IsNullOrWhiteSpace(clipper)) + await sw.WriteLineAsync($@"Clipped by: {SanitizeKeyValue(clipper)}\"); + await sw.WriteLineAsync(@$"Created at: {SanitizeKeyValue(createdAt.ToString("u"))}\"); + await sw.WriteLineAsync(@$"Video id: {SanitizeKeyValue(id)}\"); await sw.WriteLineAsync(@$"Views: {viewCount}"); } @@ -99,6 +111,27 @@ private static async Task SerializeChapters(StreamWriter sw, IEnumerable Date: Sat, 10 Aug 2024 22:25:51 -0400 Subject: [PATCH 3/4] Remove redundant flushes --- TwitchDownloaderCore/ClipDownloader.cs | 2 +- TwitchDownloaderCore/Tools/FfmpegMetadata.cs | 9 ++------- TwitchDownloaderCore/VideoDownloader.cs | 4 ++-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/TwitchDownloaderCore/ClipDownloader.cs b/TwitchDownloaderCore/ClipDownloader.cs index 7068d4c0..0130f5fe 100644 --- a/TwitchDownloaderCore/ClipDownloader.cs +++ b/TwitchDownloaderCore/ClipDownloader.cs @@ -224,7 +224,7 @@ private async Task EncodeClipWithMetadata(string inputFile, string destinationFi Process process = null; try { - await FfmpegMetadata.SerializeAsync(metadataFile, downloadOptions.Id, clipMetadata, new[] { clipChapter }, cancellationToken); + await FfmpegMetadata.SerializeAsync(metadataFile, downloadOptions.Id, clipMetadata, new[] { clipChapter }); process = new Process { diff --git a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs index 8e92cd48..8823d193 100644 --- a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs +++ b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs @@ -4,7 +4,6 @@ using System.IO; using System.Linq; using System.Text; -using System.Threading; using System.Threading.Tasks; using TwitchDownloaderCore.TwitchObjects.Gql; @@ -15,7 +14,7 @@ public static class FfmpegMetadata { private const string LINE_FEED = "\u000A"; - public static async Task SerializeAsync(string filePath, string videoId, VideoInfo videoInfo, TimeSpan startOffset, TimeSpan videoLength, IEnumerable videoMomentEdges, CancellationToken cancellationToken = default) + public static async Task SerializeAsync(string filePath, string videoId, VideoInfo videoInfo, TimeSpan startOffset, TimeSpan videoLength, IEnumerable videoMomentEdges) { await using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED }; @@ -23,13 +22,11 @@ public static async Task SerializeAsync(string filePath, string videoId, VideoIn var streamer = GetUserName(videoInfo.owner.displayName, videoInfo.owner.login); var description = videoInfo.description?.Replace(" \n", "\n").Replace("\n\n", "\n").TrimEnd(); await SerializeGlobalMetadata(sw, streamer, videoId, videoInfo.title, videoInfo.createdAt, videoInfo.viewCount, description, videoInfo.game?.displayName); - await fs.FlushAsync(cancellationToken); await SerializeChapters(sw, videoMomentEdges, startOffset, videoLength); - await fs.FlushAsync(cancellationToken); } - public static async Task SerializeAsync(string filePath, string videoId, Clip clip, IEnumerable videoMomentEdges, CancellationToken cancellationToken = default) + public static async Task SerializeAsync(string filePath, string videoId, Clip clip, IEnumerable videoMomentEdges) { await using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED }; @@ -37,10 +34,8 @@ public static async Task SerializeAsync(string filePath, string videoId, Clip cl var streamer = GetUserName(clip.broadcaster.displayName, clip.broadcaster.login); var clipper = GetUserName(clip.curator.displayName, clip.curator.login); await SerializeGlobalMetadata(sw, streamer, videoId, clip.title, clip.createdAt, clip.viewCount, game: clip.game?.displayName, clipper: clipper); - await fs.FlushAsync(cancellationToken); await SerializeChapters(sw, videoMomentEdges); - await fs.FlushAsync(cancellationToken); } private static async Task SerializeGlobalMetadata(StreamWriter sw, [AllowNull] string streamer, string id, string title, DateTime createdAt, int viewCount, [AllowNull] string description = null, [AllowNull] string game = null, diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index 31877a77..b32d10f6 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -105,8 +105,8 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF _progress.SetTemplateStatus("Finalizing Video {0}% [4/4]", 0); string metadataPath = Path.Combine(downloadFolder, "metadata.txt"); - await FfmpegMetadata.SerializeAsync(metadataPath, downloadOptions.Id.ToString(), videoInfo, downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : TimeSpan.Zero, - videoLength, videoChapterResponse.data.video.moments.edges, cancellationToken); + await FfmpegMetadata.SerializeAsync(metadataPath, downloadOptions.Id.ToString(), videoInfo, downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : TimeSpan.Zero, videoLength, + videoChapterResponse.data.video.moments.edges); var concatListPath = Path.Combine(downloadFolder, "concat.txt"); await FfmpegConcatList.SerializeAsync(concatListPath, playlist, videoListCrop, cancellationToken); From 934e0314af34bc279919f45615001d4b39e2ebc0 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sat, 10 Aug 2024 22:29:04 -0400 Subject: [PATCH 4/4] Rename SanitizeKeyValue to EscapeMetadataValue --- TwitchDownloaderCore/Tools/FfmpegMetadata.cs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs index 8823d193..7d29df09 100644 --- a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs +++ b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs @@ -43,9 +43,9 @@ private static async Task SerializeGlobalMetadata(StreamWriter sw, [AllowNull] s { // ReSharper disable once StringLiteralTypo await sw.WriteLineAsync(";FFMETADATA1"); - await sw.WriteLineAsync($"title={SanitizeKeyValue(title)} ({SanitizeKeyValue(id)})"); + await sw.WriteLineAsync($"title={EscapeMetadataValue(title)} ({EscapeMetadataValue(id)})"); if (!string.IsNullOrWhiteSpace(streamer)) - await sw.WriteLineAsync($"artist={SanitizeKeyValue(streamer)}"); + await sw.WriteLineAsync($"artist={EscapeMetadataValue(streamer)}"); await sw.WriteLineAsync($"date={createdAt:yyyy}"); // The 'date' key becomes 'year' in most formats if (!string.IsNullOrWhiteSpace(game)) await sw.WriteLineAsync($"genre={game}"); @@ -53,13 +53,13 @@ private static async Task SerializeGlobalMetadata(StreamWriter sw, [AllowNull] s if (!string.IsNullOrWhiteSpace(description)) { // We could use the 'description' key, but so few media players support mp4 descriptions that users would probably think it was missing - await sw.WriteLineAsync(@$"{SanitizeKeyValue(description.TrimEnd())}\"); + await sw.WriteLineAsync(@$"{EscapeMetadataValue(description.TrimEnd())}\"); await sw.WriteLineAsync(@"------------------------\"); } if (!string.IsNullOrWhiteSpace(clipper)) - await sw.WriteLineAsync($@"Clipped by: {SanitizeKeyValue(clipper)}\"); - await sw.WriteLineAsync(@$"Created at: {SanitizeKeyValue(createdAt.ToString("u"))}\"); - await sw.WriteLineAsync(@$"Video id: {SanitizeKeyValue(id)}\"); + await sw.WriteLineAsync($@"Clipped by: {EscapeMetadataValue(clipper)}\"); + await sw.WriteLineAsync(@$"Created at: {EscapeMetadataValue(createdAt.ToString("u"))}\"); + await sw.WriteLineAsync(@$"Video id: {EscapeMetadataValue(id)}\"); await sw.WriteLineAsync(@$"Views: {viewCount}"); } @@ -102,7 +102,7 @@ private static async Task SerializeChapters(StreamWriter sw, IEnumerable