From c60309720b8c4d8fde330cb5a8ef662c18c26d9d Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sat, 28 Oct 2023 18:27:33 +0200 Subject: [PATCH 01/31] #tsmerge Update README.md --- TwitchDownloaderCLI/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/TwitchDownloaderCLI/README.md b/TwitchDownloaderCLI/README.md index 15ba5328..1c995210 100644 --- a/TwitchDownloaderCLI/README.md +++ b/TwitchDownloaderCLI/README.md @@ -2,6 +2,7 @@ A cross platform command line tool that can do the main functions of the GUI program, which can download VODs/Clips/Chats and render chats. - [TwitchDownloaderCLI](#twitchdownloadercli) + - [Arguments for mode tsmerge](#arguments-for-mode-tsmerge) - [Arguments for mode videodownload](#arguments-for-mode-videodownload) - [Arguments for mode clipdownload](#arguments-for-mode-clipdownload) - [Arguments for mode chatdownload](#arguments-for-mode-chatdownload) @@ -14,6 +15,18 @@ A cross platform command line tool that can do the main functions of the GUI pro --- +## Arguments for mode tsmerge +Concatenates (not binary) .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file + +**-l / --inputlist (REQUIRED)** +Path to input list file in text format (one part per line). + +**-o / --output (REQUIRED)** +File the program will output to. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + ## Arguments for mode videodownload Downloads a stream VOD or highlight from Twitch @@ -348,6 +361,10 @@ Other = `1`, Broadcaster = `2`, Moderator = `4`, VIP = `8`, Subscriber = `16`, P ## Example Commands Examples of typical use cases +Concatenate several ts parts into another, keeping all streams well formatted + + TwitchDownloaderCLI tsmerge -l list.txt -o output.ts + Download a VOD with defaults TwitchDownloaderCLI videodownload --id 612942303 -o video.mp4 @@ -395,3 +412,5 @@ Default true boolean flags must be assigned: `--default-true-flag=false`. Defaul For Linux users, ensure both `fontconfig` and `libfontconfig1` are installed. `apt-get install fontconfig libfontconfig1` on Ubuntu. Some distros, like Linux Alpine, lack fonts for some languages (Arabic, Persian, Thai, etc.) If this is the case for you, install additional fonts families such as [Noto](https://fonts.google.com/noto/specimen/Noto+Sans) or check your distro's wiki page on fonts as it may have an install command for this specific scenario, such as the [Linux Alpine](https://wiki.alpinelinux.org/wiki/Fonts) font page. + +The list file for tsmerge can contain relative or absolute paths, but the format differs from Windows to Linux/UNIX/MacOS (like drive paths or "/" instead of "\" to separate directories). From baf227118256af88e943c0055239427b954a5b3e Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sat, 28 Oct 2023 18:29:17 +0200 Subject: [PATCH 02/31] #tsmerge Update Program.cs --- TwitchDownloaderCLI/Program.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/TwitchDownloaderCLI/Program.cs b/TwitchDownloaderCLI/Program.cs index 93b810d7..c09460dc 100644 --- a/TwitchDownloaderCLI/Program.cs +++ b/TwitchDownloaderCLI/Program.cs @@ -1,4 +1,4 @@ -using CommandLine; +using CommandLine; using System; using System.Collections.Generic; using System.IO; @@ -24,13 +24,14 @@ private static void Main(string[] args) config.HelpWriter = TextWriter.Null; }); - var parserResult = parser.ParseArguments(preParsedArgs); + var parserResult = parser.ParseArguments(preParsedArgs); parserResult.WithNotParsed(errors => WriteHelpText(errors, parserResult, parser.Settings)); CoreLicensor.EnsureFilesExist(AppContext.BaseDirectory); WriteApplicationBanner((ITwitchDownloaderArgs)parserResult.Value, args); parserResult + .WithParsed(MergeTS.Merge) .WithParsed(DownloadVideo.Download) .WithParsed(DownloadClip.Download) .WithParsed(DownloadChat.Download) @@ -82,4 +83,4 @@ private static void WriteApplicationBanner(ITwitchDownloaderArgs args, string[] Console.WriteLine($"{HeadingInfo.Default} {CopyrightInfo.Default.ToString()!.Replace("\u00A9", "(c)")}"); } } -} \ No newline at end of file +} From 9fc7c74a2e56e88e4d9b55c905c26ceae304c666 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sat, 28 Oct 2023 18:32:30 +0200 Subject: [PATCH 03/31] #tsmerge Add MergeTS.cs --- TwitchDownloaderCLI/Modes/MergeTS.cs | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 TwitchDownloaderCLI/Modes/MergeTS.cs diff --git a/TwitchDownloaderCLI/Modes/MergeTS.cs b/TwitchDownloaderCLI/Modes/MergeTS.cs new file mode 100644 index 00000000..2d65a7a4 --- /dev/null +++ b/TwitchDownloaderCLI/Modes/MergeTS.cs @@ -0,0 +1,35 @@ +using System; +using System.IO; +using System.Threading; +using TwitchDownloaderCLI.Modes.Arguments; +using TwitchDownloaderCLI.Tools; +using TwitchDownloaderCore; +using TwitchDownloaderCore.Options; +using TwitchDownloaderCore.Tools; + +namespace TwitchDownloaderCLI.Modes +{ + internal static class MergeTS + { + internal static void Merge(TsMergeArgs inputOptions) + { + Progress progress = new(); + progress.ProgressChanged += ProgressHandler.Progress_ProgressChanged; + + var mergeOptions = GetMergeOptions(inputOptions); + TsMerger tsMerger = new(mergeOptions, progress); + tsMerger.MergeAsync(new CancellationToken()).Wait(); + } + + private static TsMergeOptions GetMergeOptions(TsMergeArgs inputOptions) + { + TsMergeOptions mergeOptions = new() + { + OutputFile = inputOptions.OutputFile, + InputList = inputOptions.InputList + }; + + return mergeOptions; + } + } +} From fefe961da7992ebe6fd692d6ec2ba3aa8e4ab002 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sat, 28 Oct 2023 18:35:22 +0200 Subject: [PATCH 04/31] #tsmerge Add TsMergeArgs.cs --- .../Modes/Arguments/TsMergeArgs.cs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs diff --git a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs new file mode 100644 index 00000000..c9c52737 --- /dev/null +++ b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs @@ -0,0 +1,17 @@ +using CommandLine; + +namespace TwitchDownloaderCLI.Modes.Arguments +{ + [Verb("tsmerge", HelpText = "Concatenates (not binary) .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file")] + public class TsMergeArgs : ITwitchDownloaderArgs + { + [Option('l', "inputlist", Required = true, HelpText = "Path to input list file in text format (one part per line).")] + public string InputList { get; set; } + + [Option('o', "output", Required = true, HelpText = "Path to output file.")] + public string OutputFile { get; set; } + + [Option("banner", Default = true, HelpText = "Displays a banner containing version and copyright information.")] + public bool? ShowBanner { get; set; } + } +} From 0a00e81feb30ced73239332209cba5fe4d8d49af Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sat, 28 Oct 2023 18:36:54 +0200 Subject: [PATCH 05/31] #tsmerge Add TsMerger.cs --- TwitchDownloaderCore/TsMerger.cs | 132 +++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 TwitchDownloaderCore/TsMerger.cs diff --git a/TwitchDownloaderCore/TsMerger.cs b/TwitchDownloaderCore/TsMerger.cs new file mode 100644 index 00000000..fa29faf2 --- /dev/null +++ b/TwitchDownloaderCore/TsMerger.cs @@ -0,0 +1,132 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using TwitchDownloaderCore.Options; +using TwitchDownloaderCore.Tools; + +namespace TwitchDownloaderCore +{ + public sealed class TsMerger + { + private readonly TsMergeOptions downloadOptions; + private readonly IProgress _progress; + + public TsMerger(TsMergeOptions tsMergeOptions, IProgress progress) + { + downloadOptions = tsMergeOptions; + _progress = progress; + } + + public async Task MergeAsync(CancellationToken cancellationToken) + { + string downloadFolder = ""; + + try + { + string InputList = downloadOptions.InputList; + List videoPartsList = System.IO.File.ReadLines(InputList).ToList(); + videoPartsList.RemoveAll(string.IsNullOrWhiteSpace); + + _progress.Report(new ProgressReport(ReportType.SameLineStatus, "Verifying Parts 0% [1/2]")); + + VerifyDownloadedParts(videoPartsList, downloadFolder, cancellationToken); + + _progress.Report(new ProgressReport() { ReportType = ReportType.NewLineStatus, Data = "Combining Parts 0% [2/2]" }); + + await CombineVideoParts(downloadFolder, videoPartsList, cancellationToken); + + _progress.Report(new ProgressReport(100)); + } + finally + { + } + } + + private void VerifyDownloadedParts(List videoParts, string downloadFolder, CancellationToken cancellationToken) + { + var failedParts = new List(); + var partCount = videoParts.Count; + var doneCount = 0; + + foreach (var part in videoParts) + { + if (!VerifyVideoPart(downloadFolder, part)) + { + failedParts.Add(part); + } + + doneCount++; + var percent = (int)(doneCount / (double)partCount * 100); + _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Verifying Parts {percent}% [1/2]")); + _progress.Report(new ProgressReport(percent)); + + cancellationToken.ThrowIfCancellationRequested(); + } + + if (failedParts.Count != 0) + { + if (failedParts.Count == videoParts.Count) + { + // Every video part returned corrupted, probably a false positive. + return; + } + + _progress.Report(new ProgressReport(ReportType.Log, $"The following parts appear to be invalid TS files: {string.Join(", ", failedParts)}")); + } + } + + private static bool VerifyVideoPart(string downloadFolder, string part) + { + const int TS_PACKET_LENGTH = 188; // MPEG TS packets are made of a header and a body: [ 4B ][ 184B ] - https://tsduck.io/download/docs/mpegts-introduction.pdf + + var partFile = Path.Combine(downloadFolder, part); + if (!File.Exists(partFile)) + { + return false; + } + + using var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.None); + var fileLength = fs.Length; + if (fileLength == 0 || fileLength % TS_PACKET_LENGTH != 0) + { + return false; + } + + return true; + } + + private async Task CombineVideoParts(string downloadFolder, List videoParts, CancellationToken cancellationToken) + { + DriveInfo outputDrive = DriveHelper.GetOutputDrive(downloadFolder); + string outputFile = Path.Combine(downloadFolder, downloadOptions.OutputFile); + + int partCount = videoParts.Count; + int doneCount = 0; + + await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.None); + foreach (var part in videoParts) + { + await DriveHelper.WaitForDrive(outputDrive, _progress, cancellationToken); + + string partFile = Path.Combine(downloadFolder, part); + if (File.Exists(partFile)) + { + await using (var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.None)) + { + await fs.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false); + } + } + + doneCount++; + int percent = (int)(doneCount / (double)partCount * 100); + _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Combining Parts {percent}% [2/2]")); + _progress.Report(new ProgressReport(percent)); + + cancellationToken.ThrowIfCancellationRequested(); + } + } + } +} From 550c265a9b840c72f0b58e3eb1f528f1f3d4ab0a Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sat, 28 Oct 2023 18:38:52 +0200 Subject: [PATCH 06/31] #tsmerge Add TsMergeOptions.cs --- TwitchDownloaderCore/Options/TsMergeOptions.cs | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 TwitchDownloaderCore/Options/TsMergeOptions.cs diff --git a/TwitchDownloaderCore/Options/TsMergeOptions.cs b/TwitchDownloaderCore/Options/TsMergeOptions.cs new file mode 100644 index 00000000..cac7df5f --- /dev/null +++ b/TwitchDownloaderCore/Options/TsMergeOptions.cs @@ -0,0 +1,8 @@ +namespace TwitchDownloaderCore.Options +{ + public class TsMergeOptions + { + public string OutputFile { get; set; } + public string InputList { get; set; } + } +} From 0946fccdc39adede8f5797e84b1ccef07f42f003 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sat, 28 Oct 2023 18:43:52 +0200 Subject: [PATCH 07/31] Update Program.cs Try to fix weird GitHub errors. From 638d5b109301a49d6769034c71dad0983457e9d1 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sat, 28 Oct 2023 19:00:22 +0200 Subject: [PATCH 08/31] #tsmerge Update README.md --- TwitchDownloaderCLI/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TwitchDownloaderCLI/README.md b/TwitchDownloaderCLI/README.md index 1c995210..a6a4898d 100644 --- a/TwitchDownloaderCLI/README.md +++ b/TwitchDownloaderCLI/README.md @@ -1,5 +1,6 @@ # TwitchDownloaderCLI A cross platform command line tool that can do the main functions of the GUI program, which can download VODs/Clips/Chats and render chats. +Also can concatenate/combine/merge Transport Stream files, either those parts downloaded with the CLI itself or from another source. - [TwitchDownloaderCLI](#twitchdownloadercli) - [Arguments for mode tsmerge](#arguments-for-mode-tsmerge) From 36cc340f913c0cbbff9d76fd55c0819327c8608e Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 29 Oct 2023 00:16:13 +0200 Subject: [PATCH 09/31] Program.cs Remove newline at the end From 8fb6a4d5f56425c7ade3dfb023836360398815bc Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 29 Oct 2023 00:45:18 +0200 Subject: [PATCH 10/31] Delete TwitchDownloaderCLI/Program.cs --- TwitchDownloaderCLI/Program.cs | 86 ---------------------------------- 1 file changed, 86 deletions(-) delete mode 100644 TwitchDownloaderCLI/Program.cs diff --git a/TwitchDownloaderCLI/Program.cs b/TwitchDownloaderCLI/Program.cs deleted file mode 100644 index c09460dc..00000000 --- a/TwitchDownloaderCLI/Program.cs +++ /dev/null @@ -1,86 +0,0 @@ -using CommandLine; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using CommandLine.Text; -using TwitchDownloaderCLI.Modes; -using TwitchDownloaderCLI.Modes.Arguments; -using TwitchDownloaderCLI.Tools; -using TwitchDownloaderCore.Tools; - -namespace TwitchDownloaderCLI -{ - internal static class Program - { - private static void Main(string[] args) - { - var preParsedArgs = PreParseArgs.Parse(args, Path.GetFileName(Environment.ProcessPath)); - - var parser = new Parser(config => - { - config.CaseInsensitiveEnumValues = true; - config.HelpWriter = TextWriter.Null; - }); - - var parserResult = parser.ParseArguments(preParsedArgs); - parserResult.WithNotParsed(errors => WriteHelpText(errors, parserResult, parser.Settings)); - - CoreLicensor.EnsureFilesExist(AppContext.BaseDirectory); - WriteApplicationBanner((ITwitchDownloaderArgs)parserResult.Value, args); - - parserResult - .WithParsed(MergeTS.Merge) - .WithParsed(DownloadVideo.Download) - .WithParsed(DownloadClip.Download) - .WithParsed(DownloadChat.Download) - .WithParsed(UpdateChat.Update) - .WithParsed(RenderChat.Render) - .WithParsed(FfmpegHandler.ParseArgs) - .WithParsed(CacheHandler.ParseArgs); - } - - private static void WriteHelpText(IEnumerable errors, ParserResult parserResult, ParserSettings parserSettings) - { - if (errors.FirstOrDefault()?.Tag == ErrorType.NoVerbSelectedError) - { - var processFileName = Path.GetFileName(Environment.ProcessPath); - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - // Some Windows users try to double click the executable - Console.WriteLine("This is a command line tool. Please open a terminal and run \"{0} help\" from there for more information.{1}Press any key to close...", - processFileName, Environment.NewLine); - Console.ReadKey(); - } - else - { - Console.WriteLine("Usage: {0} [VERB] [OPTIONS]{1}Try \'{2} help\' for more information.", - processFileName, Environment.NewLine, processFileName); - } - } - else - { - Console.Error.WriteLine( - HelpText.AutoBuild(parserResult, builder => - { - builder.MaximumDisplayWidth = parserSettings.MaximumDisplayWidth; - builder.Copyright = CopyrightInfo.Default.ToString()!.Replace("\u00A9", "(c)"); - return builder; - })); - } - - Environment.Exit(1); - } - - private static void WriteApplicationBanner(ITwitchDownloaderArgs args, string[] argsArray) - { - if (args.ShowBanner == false || argsArray.Contains("--silent")) - { - return; - } - - Console.WriteLine($"{HeadingInfo.Default} {CopyrightInfo.Default.ToString()!.Replace("\u00A9", "(c)")}"); - } - } -} From 68d686b9e77bb4e2d635d48ab8875b3cb6cc8ce5 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 29 Oct 2023 00:46:04 +0200 Subject: [PATCH 11/31] Try to upload again Program.cs --- TwitchDownloaderCLI/Program.cs | 86 ++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 TwitchDownloaderCLI/Program.cs diff --git a/TwitchDownloaderCLI/Program.cs b/TwitchDownloaderCLI/Program.cs new file mode 100644 index 00000000..419f5dcb --- /dev/null +++ b/TwitchDownloaderCLI/Program.cs @@ -0,0 +1,86 @@ +using CommandLine; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using CommandLine.Text; +using TwitchDownloaderCLI.Modes; +using TwitchDownloaderCLI.Modes.Arguments; +using TwitchDownloaderCLI.Tools; +using TwitchDownloaderCore.Tools; + +namespace TwitchDownloaderCLI +{ + internal static class Program + { + private static void Main(string[] args) + { + var preParsedArgs = PreParseArgs.Parse(args, Path.GetFileName(Environment.ProcessPath)); + + var parser = new Parser(config => + { + config.CaseInsensitiveEnumValues = true; + config.HelpWriter = TextWriter.Null; + }); + + var parserResult = parser.ParseArguments(preParsedArgs); + parserResult.WithNotParsed(errors => WriteHelpText(errors, parserResult, parser.Settings)); + + CoreLicensor.EnsureFilesExist(AppContext.BaseDirectory); + WriteApplicationBanner((ITwitchDownloaderArgs)parserResult.Value, args); + + parserResult + .WithParsed(MergeTS.Merge) + .WithParsed(DownloadVideo.Download) + .WithParsed(DownloadClip.Download) + .WithParsed(DownloadChat.Download) + .WithParsed(UpdateChat.Update) + .WithParsed(RenderChat.Render) + .WithParsed(FfmpegHandler.ParseArgs) + .WithParsed(CacheHandler.ParseArgs); + } + + private static void WriteHelpText(IEnumerable errors, ParserResult parserResult, ParserSettings parserSettings) + { + if (errors.FirstOrDefault()?.Tag == ErrorType.NoVerbSelectedError) + { + var processFileName = Path.GetFileName(Environment.ProcessPath); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + // Some Windows users try to double click the executable + Console.WriteLine("This is a command line tool. Please open a terminal and run \"{0} help\" from there for more information.{1}Press any key to close...", + processFileName, Environment.NewLine); + Console.ReadKey(); + } + else + { + Console.WriteLine("Usage: {0} [VERB] [OPTIONS]{1}Try \'{2} help\' for more information.", + processFileName, Environment.NewLine, processFileName); + } + } + else + { + Console.Error.WriteLine( + HelpText.AutoBuild(parserResult, builder => + { + builder.MaximumDisplayWidth = parserSettings.MaximumDisplayWidth; + builder.Copyright = CopyrightInfo.Default.ToString()!.Replace("\u00A9", "(c)"); + return builder; + })); + } + + Environment.Exit(1); + } + + private static void WriteApplicationBanner(ITwitchDownloaderArgs args, string[] argsArray) + { + if (args.ShowBanner == false || argsArray.Contains("--silent")) + { + return; + } + + Console.WriteLine($"{HeadingInfo.Default} {CopyrightInfo.Default.ToString()!.Replace("\u00A9", "(c)")}"); + } + } +} \ No newline at end of file From 32e6b0d98c06090ff73c00376e6f2cf36738446e Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:00:53 +0100 Subject: [PATCH 12/31] Delete TwitchDownloaderCLI/README.md --- TwitchDownloaderCLI/README.md | 417 ---------------------------------- 1 file changed, 417 deletions(-) delete mode 100644 TwitchDownloaderCLI/README.md diff --git a/TwitchDownloaderCLI/README.md b/TwitchDownloaderCLI/README.md deleted file mode 100644 index a6a4898d..00000000 --- a/TwitchDownloaderCLI/README.md +++ /dev/null @@ -1,417 +0,0 @@ -# TwitchDownloaderCLI -A cross platform command line tool that can do the main functions of the GUI program, which can download VODs/Clips/Chats and render chats. -Also can concatenate/combine/merge Transport Stream files, either those parts downloaded with the CLI itself or from another source. - -- [TwitchDownloaderCLI](#twitchdownloadercli) - - [Arguments for mode tsmerge](#arguments-for-mode-tsmerge) - - [Arguments for mode videodownload](#arguments-for-mode-videodownload) - - [Arguments for mode clipdownload](#arguments-for-mode-clipdownload) - - [Arguments for mode chatdownload](#arguments-for-mode-chatdownload) - - [Arguments for mode chatupdate](#arguments-for-mode-chatupdate) - - [Arguments for mode chatrender](#arguments-for-mode-chatrender) - - [Arguments for mode ffmpeg](#arguments-for-mode-ffmpeg) - - [Arguments for mode cache](#arguments-for-mode-cache) - - [Example Commands](#example-commands) - - [Additional Notes](#additional-notes) - ---- - -## Arguments for mode tsmerge -Concatenates (not binary) .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file - -**-l / --inputlist (REQUIRED)** -Path to input list file in text format (one part per line). - -**-o / --output (REQUIRED)** -File the program will output to. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - -## Arguments for mode videodownload -Downloads a stream VOD or highlight from Twitch - -**-u / --id (REQUIRED)** -The ID or URL of the VOD to download. - -**-o / --output (REQUIRED)** -File the program will output to. - -**-q / --quality** -The quality the program will attempt to download, for example "1080p60", if not found will download highest quality stream. - -**-b / --beginning** -Time in seconds to crop beginning. For example if I had a 10 second stream but only wanted the last 7 seconds of it I would use `-b 3` to skip the first 3 seconds. - -**-e / --ending** -Time in seconds to crop ending. For example if I had a 10 second stream but only wanted the first 4 seconds of it I would use `-e 4` to end on the 4th second. - -Extra example, if I wanted only seconds 3-6 in a 10 second stream I would do `-b 3 -e 6` - -**-t / --threads** -(Default: `4`) Number of download threads. - -**--bandwidth** -(Default: `-1`) The maximum bandwidth a thread will be allowed to use in kibibytes per second (KiB/s), or `-1` for no maximum. - -**--oauth** -OAuth access token to download subscriber only VODs. **DO NOT SHARE YOUR OAUTH TOKEN WITH ANYONE.** - -**--ffmpeg-path** -Path to FFmpeg executable. - -**--temp-path** -Path to temporary folder for cache. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - -## Arguments for mode clipdownload -Downloads a clip from Twitch - -**-u / --id (REQUIRED)** -The ID or URL of the Clip to download. - -**-o / --output (REQUIRED)** -File the program will output to. - -**-q / --quality** -The quality the program will attempt to download, for example "1080p60", if not found will download highest quality video. - -**--bandwidth** -(Default: `-1`) The maximum bandwidth the clip downloader is allowed to use in kibibytes per second (KiB/s), or `-1` for no maximum. - -**--encode-metadata** -(Default: `true`) Uses FFmpeg to add metadata to the clip output file. - -**--ffmpeg-path** -Path to FFmpeg executable. - -**--temp-path** -Path to temporary folder for cache. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - -## Arguments for mode chatdownload -Downloads the chat of a VOD, highlight, or clip - -**-u / --id (REQUIRED)** -The ID or URL of the VOD or clip to download. - -**-o / --output (REQUIRED)** -File the program will output to. File extension will be used to determine download type. Valid extensions are: `.json`, `.html`, and `.txt`. - -**--compression** -(Default: `None`) Compresses an output json chat file using a specified compression, usually resulting in 40-90% size reductions. Valid values are: `None`, `Gzip`. More formats will be supported in the future. - -**-b / --beginning** -Time in seconds to crop beginning. For example if I had a 10 second stream but only wanted the last 7 seconds of it I would use `-b 3` to skip the first 3 seconds. - -**-e / --ending** -Time in seconds to crop ending. For example if I had a 10 second stream but only wanted the first 4 seconds of it I would use `-e 4` to end on the 4th second. - -**-E / --embed-images** -(Default: `false`) Embed first party emotes, badges, and cheermotes into the download file for offline rendering. Useful for archival purposes, file size will be larger. - -**--bttv** -(Default: `true`) BTTV emote embedding. Requires `-E / --embed-images`. - -**--ffz** -(Default: `true`) FFZ emote embedding. Requires `-E / --embed-images`. - -**--stv** -(Default: `true`) 7TV emote embedding. Requires `-E / --embed-images`. - -**--timestamp-format** -(Default: `Relative`) Sets the timestamp format for .txt chat logs. Valid values are: `Utc`, `UtcFull`, `Relative`, and `None`. - -**--chat-connections** -(Default: `4`) The number of parallel downloads for chat. - -**--silent** -(Default: `false`) Suppresses progress console output. - -**--temp-path** -Path to temporary folder for cache. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - -## Arguments for mode chatupdate -Updates the embedded emotes, badges, bits, and crops of a chat download and/or converts a JSON chat to another format - -**-i / --input (REQUIRED)** -Path to input file. Valid extensions are: `.json`, `.json.gz`. - -**-o / --output (REQUIRED)** -Path to output file. File extension will be used to determine new chat type. Valid extensions are: `.json`, `.html`, and `.txt`. - -**-c / --compression** -(Default: `None`) Compresses an output json chat file using a specified compression, usually resulting in 40-90% size reductions. Valid values are: `None`, `Gzip`. More formats will be supported in the future. - -**-E / --embed-missing** -(Default: `false`) Embed missing emotes, badges, and cheermotes. Already embedded images will be untouched. - -**-R / --replace-embeds** -(Default: `false`) Replace all embedded emotes, badges, and cheermotes in the file. All embedded data will be overwritten! - -**b / --beginning** -(Default: `-1`) New time in seconds for chat beginning. Comments may be added but not removed. -1 = No crop. - -**-e / --ending** -(Default: `-1`) New time in seconds for chat beginning. Comments may be added but not removed. -1 = No crop. - -**--bttv** -(Default: `true`) Enable embedding BTTV emotes. - -**--ffz** -(Default: `true`) Enable embedding FFZ emotes. - -**--stv** -(Default: `true`) Enable embedding 7TV emotes. - -**--timestamp-format** -(Default: `Relative`) Sets the timestamp format for .txt chat logs. Valid values are: `Utc`, `Relative`, and `None`. - -**--temp-path** -Path to temporary folder for cache. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - -## Arguments for mode chatrender -Renders a chat JSON as a video - -**-i / --input (REQUIRED)** -The path to the `.json` or `.json.gz` chat file input. - -**-o / --output (REQUIRED)** -File the program will output to. - -**--background-color** -(Default: `#111111`) The render background color in the string format of `#RRGGBB` or `#AARRGGBB` in hexadecimal. - -**--alt-background-color** -(Default: `#191919`) The alternate message background color in the string format of `#RRGGBB` or `#AARRGGBB` in hexadecimal. Requires `--alternate-backgrounds`. - -**--message-color** -(Default: `#ffffff`) The message text color in the string format of `#RRGGBB` or `#AARRGGBB` in hexadecimal. - -**-w / --chat-width** -(Default: `350`) Width of chat render. - -**-h / --chat-height** -(Default: `600`) Height of chat render. - -**-b / --beginning** -(Default: `-1`) Time in seconds to crop the beginning of the render. - -**-e / --ending** -(Default: `-1`) Time in seconds to crop the ending of the render. - -**--bttv** -(Default: `true`) Enable BTTV emotes. - -**--ffz** -(Default: `true`) Enable FFZ emotes. - -**--stv** -(Default: `true`) Enable 7TV emotes. - -**--allow-unlisted-emotes** -(Default: `true`) Allow unlisted 7TV emotes in the render. - -**--sub-messages** -(Default: `true`) Enable sub / re-sub messages. - -**--badges** -(Default: `true`) Enable chat badges. - -**--outline** -(Default: `false`) Enable outline around chat messages. - -**--outline-size** -(Default: `4`) Size of outline if outline is enabled. - -**-f / --font** -(Default: `Inter Embedded`) Font to use. - -**--font-size** -(Default: `12`) Font size. - -**--message-fontstyle** -(Default: `normal`) Font style of message. Valid values are **normal**, **bold**, and **italic**. - -**--username-fontstyle** -(Default: `bold`) Font style of username. Valid values are **normal**, **bold**, and **italic**. - -**--timestamp** -(Default: `false`) Enables timestamps to left of messages, similar to VOD chat on Twitch. - -**--generate-mask** -(Default: `false`) Generates a mask file of the chat in addition to the rendered chat. - -**--sharpening** -(Default: `false`) Appends `-filter_complex "smartblur=lr=1:ls=-1.0"` to the `input-args`. Works best with `font-size` 24 or larger. - -**--framerate** -(Default: `30`) Framerate of the render. - -**--update-rate** -(Default: `0.2`) Time in seconds to update chat render output. - -**--input-args** -(Default: `-framerate {fps} -f rawvideo -analyzeduration {max_int} -probesize {max_int} -pix_fmt bgra -video_size {width}x{height} -i -`) Input arguments for FFmpeg chat render. - -**--output-args** -(Default: `-c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "{save_path}"`) Output arguments for FFmpeg chat render. - -**--ignore-users** -(Default: ` `) List of usernames to ignore when rendering, separated by commas. Not case-sensitive. - -**--ban-words** -(Default: ` `) List of words or phrases to ignore when rendering, separated by commas. Not case-sensitive. - -**--badge-filter** -(Default: `0`) Bitmask of types of Chat Badges to filter out. Add the numbers of the types of badges you want to filter. For example, to filter out Moderator and Broadcaster badges only enter the value of 6. - -Other = `1`, Broadcaster = `2`, Moderator = `4`, VIP = `8`, Subscriber = `16`, Predictions = `32`, NoAudioVisual = `64`, PrimeGaming = `128` - -**--dispersion** -(Default: `false`) In November 2022 a Twitch API change made chat messages download only in whole seconds. This option uses additional metadata to attempt to restore messages to when they were actually sent. This may result in a different comment order. Requires an update rate less than 1.0 for effective results. - -**--alternate-backgrounds** -(Default: `false`) Alternates the background color of every other chat message to help tell them apart. - -**--offline** -(Default: `false`) Render completely offline using only embedded emotes, badges, and bits from the input json. - -**--emoji-vendor** -(Default: `notocolor`) The emoji vendor used for rendering emojis. Valid values are: `twitter` / `twemoji`, `google` / `notocolor`, `none`. - -**--ffmpeg-path** -(Default: ` `) Path to FFmpeg executable. - -**--temp-path** -(Default: ` `) Path to temporary folder for cache. - -**--verbose-ffmpeg** -(Default: `false`) Prints every message from FFmpeg. - -**--skip-drive-waiting** -(Default: `false`) Do not wait for the output drive to transmit a ready signal before writing the next frame. Waiting is usually only necessary on low-end USB drives. Skipping can result in 1-5% render speed increases. - -**--scale-emote** -(Default: `1.0`) Number to scale emote images. - -**--scale-badge** -(Default: `1.0`) Number to scale badge images. - -**--scale-emoji** -(Default: `1.0`) Number to scale emoji images. - -**--scale-vertical** -(Default: `1.0`) Number to scale vertical padding. - -**--scale-side-padding** -(Default: `1.0`) Number to scale side padding. - -**--scale-section-height** -(Default: `1.0`) Number to scale section height of comments. - -**--scale-word-space** -(Default: `1.0`) Number to scale spacing between words. - -**--scale-emote-space** -(Default: `1.0`) Number to scale spacing between emotes. - -**--scale-highlight-stroke** -(Default: `1.0`) Number to scale highlight stroke size (sub messages). - -**--scale-highlight-indent** -(Default: `1.0`) Number to scale highlight indent size (sub messages). - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - - -## Arguments for mode ffmpeg -Manage standalone FFmpeg - -**-d / --download** -(Default: `false`) Downloads FFmpeg as a standalone file. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - -## Arguments for mode cache -Manage the working cache. - -**-c / --clear** -(Default: `false`) Clears the default cache folder. - -**--force-clear** -(Default: `false`) Clears the default cache folder, bypassing the confirmation prompt. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - ---- - -## Example Commands -Examples of typical use cases - -Concatenate several ts parts into another, keeping all streams well formatted - - TwitchDownloaderCLI tsmerge -l list.txt -o output.ts - -Download a VOD with defaults - - TwitchDownloaderCLI videodownload --id 612942303 -o video.mp4 - -Download a Clip with defaults - - TwitchDownloaderCLI clipdownload --id NurturingCalmHamburgerVoHiYo -o clip.mp4 - -Download a Chat JSON with embedded emotes/badges from Twitch and emotes from Bttv - - TwitchDownloaderCLI chatdownload --id 612942303 --embed-images --bttv=true --ffz=false --stv=false -o chat.json - -Download a Chat as plain text with timestamps - - TwitchDownloaderCLI chatdownload --id 612942303 --timestamp-format Relative -o chat.txt - -Add embeds to a chat file that was downloaded without embeds - - TwitchDownloaderCLI chatupdate -i chat.json -o chat_embedded.json --embed-missing - -Convert a JSON chat file to HTML - - TwitchDownloaderCLI chatupdate -i chat.json -o chat.html - -Render a chat with defaults - - TwitchDownloaderCLI chatrender -i chat.json -o chat.mp4 - -Render a chat with custom video settings and message outlines - - TwitchDownloaderCLI chatrender -i chat.json -h 1440 -w 720 --framerate 60 --outline -o chat.mp4 - -Render a chat with custom FFmpeg arguments - - TwitchDownloaderCLI chatrender -i chat.json --output-args='-c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "{save_path}"' -o chat.mp4 - ---- - -## Additional Notes - -String arguments, such as output file, that contain spaces should be wrapped in either single quotes ' or double quotes " . - -Default true boolean flags must be assigned: `--default-true-flag=false`. Default false boolean flags should still be raised normally: `--default-false-flag` - -For Linux users, ensure both `fontconfig` and `libfontconfig1` are installed. `apt-get install fontconfig libfontconfig1` on Ubuntu. - -Some distros, like Linux Alpine, lack fonts for some languages (Arabic, Persian, Thai, etc.) If this is the case for you, install additional fonts families such as [Noto](https://fonts.google.com/noto/specimen/Noto+Sans) or check your distro's wiki page on fonts as it may have an install command for this specific scenario, such as the [Linux Alpine](https://wiki.alpinelinux.org/wiki/Fonts) font page. - -The list file for tsmerge can contain relative or absolute paths, but the format differs from Windows to Linux/UNIX/MacOS (like drive paths or "/" instead of "\" to separate directories). From a1922072cccc6e1076826c1e64f512700b6bee2e Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:01:16 +0100 Subject: [PATCH 13/31] Add files via upload --- TwitchDownloaderCLI/README.md | 420 ++++++++++++++++++++++++++++++++++ 1 file changed, 420 insertions(+) create mode 100644 TwitchDownloaderCLI/README.md diff --git a/TwitchDownloaderCLI/README.md b/TwitchDownloaderCLI/README.md new file mode 100644 index 00000000..bbfb75a8 --- /dev/null +++ b/TwitchDownloaderCLI/README.md @@ -0,0 +1,420 @@ +# TwitchDownloaderCLI +A cross platform command line tool that can do the main functions of the GUI program, which can download VODs/Clips/Chats and render chats. +Also can concatenate/combine/merge Transport Stream files, either those parts downloaded with the CLI itself or from another source. + +- [TwitchDownloaderCLI](#twitchdownloadercli) + - [Arguments for mode videodownload](#arguments-for-mode-videodownload) + - [Arguments for mode clipdownload](#arguments-for-mode-clipdownload) + - [Arguments for mode chatdownload](#arguments-for-mode-chatdownload) + - [Arguments for mode chatupdate](#arguments-for-mode-chatupdate) + - [Arguments for mode chatrender](#arguments-for-mode-chatrender) + - [Arguments for mode ffmpeg](#arguments-for-mode-ffmpeg) + - [Arguments for mode cache](#arguments-for-mode-cache) + - [Arguments for mode tsmerge](#arguments-for-mode-tsmerge) + - [Example Commands](#example-commands) + - [Additional Notes](#additional-notes) + +--- + +## Arguments for mode videodownload +Downloads a stream VOD or highlight from Twitch + +**-u / --id (REQUIRED)** +The ID or URL of the VOD to download. + +**-o / --output (REQUIRED)** +File the program will output to. + +**-q / --quality** +The quality the program will attempt to download, for example "1080p60", if not found will download highest quality stream. + +**-b / --beginning** +Time in seconds to crop beginning. For example if I had a 10 second stream but only wanted the last 7 seconds of it I would use `-b 3` to skip the first 3 seconds. + +**-e / --ending** +Time in seconds to crop ending. For example if I had a 10 second stream but only wanted the first 4 seconds of it I would use `-e 4` to end on the 4th second. + +Extra example, if I wanted only seconds 3-6 in a 10 second stream I would do `-b 3 -e 6` + +**-t / --threads** +(Default: `4`) Number of download threads. + +**--bandwidth** +(Default: `-1`) The maximum bandwidth a thread will be allowed to use in kibibytes per second (KiB/s), or `-1` for no maximum. + +**--oauth** +OAuth access token to download subscriber only VODs. **DO NOT SHARE YOUR OAUTH TOKEN WITH ANYONE.** + +**--ffmpeg-path** +Path to FFmpeg executable. + +**--temp-path** +Path to temporary folder for cache. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +## Arguments for mode clipdownload +Downloads a clip from Twitch + +**-u / --id (REQUIRED)** +The ID or URL of the Clip to download. + +**-o / --output (REQUIRED)** +File the program will output to. + +**-q / --quality** +The quality the program will attempt to download, for example "1080p60", if not found will download highest quality video. + +**--bandwidth** +(Default: `-1`) The maximum bandwidth the clip downloader is allowed to use in kibibytes per second (KiB/s), or `-1` for no maximum. + +**--encode-metadata** +(Default: `true`) Uses FFmpeg to add metadata to the clip output file. + +**--ffmpeg-path** +Path to FFmpeg executable. + +**--temp-path** +Path to temporary folder for cache. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +## Arguments for mode chatdownload +Downloads the chat of a VOD, highlight, or clip + +**-u / --id (REQUIRED)** +The ID or URL of the VOD or clip to download. + +**-o / --output (REQUIRED)** +File the program will output to. File extension will be used to determine download type. Valid extensions are: `.json`, `.html`, and `.txt`. + +**--compression** +(Default: `None`) Compresses an output json chat file using a specified compression, usually resulting in 40-90% size reductions. Valid values are: `None`, `Gzip`. More formats will be supported in the future. + +**-b / --beginning** +Time in seconds to crop beginning. For example if I had a 10 second stream but only wanted the last 7 seconds of it I would use `-b 3` to skip the first 3 seconds. + +**-e / --ending** +Time in seconds to crop ending. For example if I had a 10 second stream but only wanted the first 4 seconds of it I would use `-e 4` to end on the 4th second. + +**-E / --embed-images** +(Default: `false`) Embed first party emotes, badges, and cheermotes into the download file for offline rendering. Useful for archival purposes, file size will be larger. + +**--bttv** +(Default: `true`) BTTV emote embedding. Requires `-E / --embed-images`. + +**--ffz** +(Default: `true`) FFZ emote embedding. Requires `-E / --embed-images`. + +**--stv** +(Default: `true`) 7TV emote embedding. Requires `-E / --embed-images`. + +**--timestamp-format** +(Default: `Relative`) Sets the timestamp format for .txt chat logs. Valid values are: `Utc`, `UtcFull`, `Relative`, and `None`. + +**--chat-connections** +(Default: `4`) The number of parallel downloads for chat. + +**--silent** +(Default: `false`) Suppresses progress console output. + +**--temp-path** +Path to temporary folder for cache. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +## Arguments for mode chatupdate +Updates the embedded emotes, badges, bits, and crops of a chat download and/or converts a JSON chat to another format + +**-i / --input (REQUIRED)** +Path to input file. Valid extensions are: `.json`, `.json.gz`. + +**-o / --output (REQUIRED)** +Path to output file. File extension will be used to determine new chat type. Valid extensions are: `.json`, `.html`, and `.txt`. + +**-c / --compression** +(Default: `None`) Compresses an output json chat file using a specified compression, usually resulting in 40-90% size reductions. Valid values are: `None`, `Gzip`. More formats will be supported in the future. + +**-E / --embed-missing** +(Default: `false`) Embed missing emotes, badges, and cheermotes. Already embedded images will be untouched. + +**-R / --replace-embeds** +(Default: `false`) Replace all embedded emotes, badges, and cheermotes in the file. All embedded data will be overwritten! + +**b / --beginning** +(Default: `-1`) New time in seconds for chat beginning. Comments may be added but not removed. -1 = No crop. + +**-e / --ending** +(Default: `-1`) New time in seconds for chat beginning. Comments may be added but not removed. -1 = No crop. + +**--bttv** +(Default: `true`) Enable embedding BTTV emotes. + +**--ffz** +(Default: `true`) Enable embedding FFZ emotes. + +**--stv** +(Default: `true`) Enable embedding 7TV emotes. + +**--timestamp-format** +(Default: `Relative`) Sets the timestamp format for .txt chat logs. Valid values are: `Utc`, `Relative`, and `None`. + +**--temp-path** +Path to temporary folder for cache. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +## Arguments for mode chatrender +Renders a chat JSON as a video + +**-i / --input (REQUIRED)** +The path to the `.json` or `.json.gz` chat file input. + +**-o / --output (REQUIRED)** +File the program will output to. + +**--background-color** +(Default: `#111111`) The render background color in the string format of `#RRGGBB` or `#AARRGGBB` in hexadecimal. + +**--alt-background-color** +(Default: `#191919`) The alternate message background color in the string format of `#RRGGBB` or `#AARRGGBB` in hexadecimal. Requires `--alternate-backgrounds`. + +**--message-color** +(Default: `#ffffff`) The message text color in the string format of `#RRGGBB` or `#AARRGGBB` in hexadecimal. + +**-w / --chat-width** +(Default: `350`) Width of chat render. + +**-h / --chat-height** +(Default: `600`) Height of chat render. + +**-b / --beginning** +(Default: `-1`) Time in seconds to crop the beginning of the render. + +**-e / --ending** +(Default: `-1`) Time in seconds to crop the ending of the render. + +**--bttv** +(Default: `true`) Enable BTTV emotes. + +**--ffz** +(Default: `true`) Enable FFZ emotes. + +**--stv** +(Default: `true`) Enable 7TV emotes. + +**--allow-unlisted-emotes** +(Default: `true`) Allow unlisted 7TV emotes in the render. + +**--sub-messages** +(Default: `true`) Enable sub / re-sub messages. + +**--badges** +(Default: `true`) Enable chat badges. + +**--outline** +(Default: `false`) Enable outline around chat messages. + +**--outline-size** +(Default: `4`) Size of outline if outline is enabled. + +**-f / --font** +(Default: `Inter Embedded`) Font to use. + +**--font-size** +(Default: `12`) Font size. + +**--message-fontstyle** +(Default: `normal`) Font style of message. Valid values are **normal**, **bold**, and **italic**. + +**--username-fontstyle** +(Default: `bold`) Font style of username. Valid values are **normal**, **bold**, and **italic**. + +**--timestamp** +(Default: `false`) Enables timestamps to left of messages, similar to VOD chat on Twitch. + +**--generate-mask** +(Default: `false`) Generates a mask file of the chat in addition to the rendered chat. + +**--sharpening** +(Default: `false`) Appends `-filter_complex "smartblur=lr=1:ls=-1.0"` to the `input-args`. Works best with `font-size` 24 or larger. + +**--framerate** +(Default: `30`) Framerate of the render. + +**--update-rate** +(Default: `0.2`) Time in seconds to update chat render output. + +**--input-args** +(Default: `-framerate {fps} -f rawvideo -analyzeduration {max_int} -probesize {max_int} -pix_fmt bgra -video_size {width}x{height} -i -`) Input arguments for FFmpeg chat render. + +**--output-args** +(Default: `-c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "{save_path}"`) Output arguments for FFmpeg chat render. + +**--ignore-users** +(Default: ` `) List of usernames to ignore when rendering, separated by commas. Not case-sensitive. + +**--ban-words** +(Default: ` `) List of words or phrases to ignore when rendering, separated by commas. Not case-sensitive. + +**--badge-filter** +(Default: `0`) Bitmask of types of Chat Badges to filter out. Add the numbers of the types of badges you want to filter. For example, to filter out Moderator and Broadcaster badges only enter the value of 6. + +Other = `1`, Broadcaster = `2`, Moderator = `4`, VIP = `8`, Subscriber = `16`, Predictions = `32`, NoAudioVisual = `64`, PrimeGaming = `128` + +**--dispersion** +(Default: `false`) In November 2022 a Twitch API change made chat messages download only in whole seconds. This option uses additional metadata to attempt to restore messages to when they were actually sent. This may result in a different comment order. Requires an update rate less than 1.0 for effective results. + +**--alternate-backgrounds** +(Default: `false`) Alternates the background color of every other chat message to help tell them apart. + +**--offline** +(Default: `false`) Render completely offline using only embedded emotes, badges, and bits from the input json. + +**--emoji-vendor** +(Default: `notocolor`) The emoji vendor used for rendering emojis. Valid values are: `twitter` / `twemoji`, `google` / `notocolor`, `none`. + +**--ffmpeg-path** +(Default: ` `) Path to FFmpeg executable. + +**--temp-path** +(Default: ` `) Path to temporary folder for cache. + +**--verbose-ffmpeg** +(Default: `false`) Prints every message from FFmpeg. + +**--skip-drive-waiting** +(Default: `false`) Do not wait for the output drive to transmit a ready signal before writing the next frame. Waiting is usually only necessary on low-end USB drives. Skipping can result in 1-5% render speed increases. + +**--scale-emote** +(Default: `1.0`) Number to scale emote images. + +**--scale-badge** +(Default: `1.0`) Number to scale badge images. + +**--scale-emoji** +(Default: `1.0`) Number to scale emoji images. + +**--scale-vertical** +(Default: `1.0`) Number to scale vertical padding. + +**--scale-side-padding** +(Default: `1.0`) Number to scale side padding. + +**--scale-section-height** +(Default: `1.0`) Number to scale section height of comments. + +**--scale-word-space** +(Default: `1.0`) Number to scale spacing between words. + +**--scale-emote-space** +(Default: `1.0`) Number to scale spacing between emotes. + +**--scale-highlight-stroke** +(Default: `1.0`) Number to scale highlight stroke size (sub messages). + +**--scale-highlight-indent** +(Default: `1.0`) Number to scale highlight indent size (sub messages). + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + + +## Arguments for mode ffmpeg +Manage standalone FFmpeg + +**-d / --download** +(Default: `false`) Downloads FFmpeg as a standalone file. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +## Arguments for mode cache +Manage the working cache. + +**-c / --clear** +(Default: `false`) Clears the default cache folder. + +**--force-clear** +(Default: `false`) Clears the default cache folder, bypassing the confirmation prompt. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +## Arguments for mode tsmerge +Concatenates .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file + +**-l / --inputlist (REQUIRED)** +Path to input list file in text format (one part per line). + +**-o / --output (REQUIRED)** +File the program will output to. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +--- + +## Example Commands +Examples of typical use cases + +Download a VOD with defaults + + TwitchDownloaderCLI videodownload --id 612942303 -o video.mp4 + +Download a Clip with defaults + + TwitchDownloaderCLI clipdownload --id NurturingCalmHamburgerVoHiYo -o clip.mp4 + +Download a Chat JSON with embedded emotes/badges from Twitch and emotes from Bttv + + TwitchDownloaderCLI chatdownload --id 612942303 --embed-images --bttv=true --ffz=false --stv=false -o chat.json + +Download a Chat as plain text with timestamps + + TwitchDownloaderCLI chatdownload --id 612942303 --timestamp-format Relative -o chat.txt + +Add embeds to a chat file that was downloaded without embeds + + TwitchDownloaderCLI chatupdate -i chat.json -o chat_embedded.json --embed-missing + +Convert a JSON chat file to HTML + + TwitchDownloaderCLI chatupdate -i chat.json -o chat.html + +Render a chat with defaults + + TwitchDownloaderCLI chatrender -i chat.json -o chat.mp4 + +Render a chat with custom video settings and message outlines + + TwitchDownloaderCLI chatrender -i chat.json -h 1440 -w 720 --framerate 60 --outline -o chat.mp4 + +Render a chat with custom FFmpeg arguments + + TwitchDownloaderCLI chatrender -i chat.json --output-args='-c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "{save_path}"' -o chat.mp4 + +Concatenate several ts parts into another + + TwitchDownloaderCLI tsmerge -l list.txt -o output.ts + +--- + +## Additional Notes + +String arguments, such as output file, that contain spaces should be wrapped in either single quotes ' or double quotes " . + +Default true boolean flags must be assigned: `--default-true-flag=false`. Default false boolean flags should still be raised normally: `--default-false-flag` + +For Linux users, ensure both `fontconfig` and `libfontconfig1` are installed. `apt-get install fontconfig libfontconfig1` on Ubuntu. + +Some distros, like Linux Alpine, lack fonts for some languages (Arabic, Persian, Thai, etc.) If this is the case for you, install additional fonts families such as [Noto](https://fonts.google.com/noto/specimen/Noto+Sans) or check your distro's wiki page on fonts as it may have an install command for this specific scenario, such as the [Linux Alpine](https://wiki.alpinelinux.org/wiki/Fonts) font page. + +The list file for tsmerge can contain relative or absolute paths, but the format differs from Windows to Linux/UNIX/MacOS (like drive paths or "/" instead of "\" to separate directories). + +The concatenation made by tsmerge is not a raw (binary) concatenation, like `dd`, `cat` or `pv` would do on Linux, or `copy /b` on Windows. +Instead, the streams have to be partially recoded, to keep its structure valid and playable. Using other software may not achieve this result. From 7a41051d9bc946287552b06f77379e1b4703077d Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:05:17 +0100 Subject: [PATCH 14/31] Delete TwitchDownloaderCLI/Program.cs --- TwitchDownloaderCLI/Program.cs | 86 ---------------------------------- 1 file changed, 86 deletions(-) delete mode 100644 TwitchDownloaderCLI/Program.cs diff --git a/TwitchDownloaderCLI/Program.cs b/TwitchDownloaderCLI/Program.cs deleted file mode 100644 index 419f5dcb..00000000 --- a/TwitchDownloaderCLI/Program.cs +++ /dev/null @@ -1,86 +0,0 @@ -using CommandLine; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using CommandLine.Text; -using TwitchDownloaderCLI.Modes; -using TwitchDownloaderCLI.Modes.Arguments; -using TwitchDownloaderCLI.Tools; -using TwitchDownloaderCore.Tools; - -namespace TwitchDownloaderCLI -{ - internal static class Program - { - private static void Main(string[] args) - { - var preParsedArgs = PreParseArgs.Parse(args, Path.GetFileName(Environment.ProcessPath)); - - var parser = new Parser(config => - { - config.CaseInsensitiveEnumValues = true; - config.HelpWriter = TextWriter.Null; - }); - - var parserResult = parser.ParseArguments(preParsedArgs); - parserResult.WithNotParsed(errors => WriteHelpText(errors, parserResult, parser.Settings)); - - CoreLicensor.EnsureFilesExist(AppContext.BaseDirectory); - WriteApplicationBanner((ITwitchDownloaderArgs)parserResult.Value, args); - - parserResult - .WithParsed(MergeTS.Merge) - .WithParsed(DownloadVideo.Download) - .WithParsed(DownloadClip.Download) - .WithParsed(DownloadChat.Download) - .WithParsed(UpdateChat.Update) - .WithParsed(RenderChat.Render) - .WithParsed(FfmpegHandler.ParseArgs) - .WithParsed(CacheHandler.ParseArgs); - } - - private static void WriteHelpText(IEnumerable errors, ParserResult parserResult, ParserSettings parserSettings) - { - if (errors.FirstOrDefault()?.Tag == ErrorType.NoVerbSelectedError) - { - var processFileName = Path.GetFileName(Environment.ProcessPath); - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - // Some Windows users try to double click the executable - Console.WriteLine("This is a command line tool. Please open a terminal and run \"{0} help\" from there for more information.{1}Press any key to close...", - processFileName, Environment.NewLine); - Console.ReadKey(); - } - else - { - Console.WriteLine("Usage: {0} [VERB] [OPTIONS]{1}Try \'{2} help\' for more information.", - processFileName, Environment.NewLine, processFileName); - } - } - else - { - Console.Error.WriteLine( - HelpText.AutoBuild(parserResult, builder => - { - builder.MaximumDisplayWidth = parserSettings.MaximumDisplayWidth; - builder.Copyright = CopyrightInfo.Default.ToString()!.Replace("\u00A9", "(c)"); - return builder; - })); - } - - Environment.Exit(1); - } - - private static void WriteApplicationBanner(ITwitchDownloaderArgs args, string[] argsArray) - { - if (args.ShowBanner == false || argsArray.Contains("--silent")) - { - return; - } - - Console.WriteLine($"{HeadingInfo.Default} {CopyrightInfo.Default.ToString()!.Replace("\u00A9", "(c)")}"); - } - } -} \ No newline at end of file From b71b6b80b170105d7fb6a16416df29335ed74017 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:05:32 +0100 Subject: [PATCH 15/31] Add files via upload --- TwitchDownloaderCLI/Program.cs | 86 ++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 TwitchDownloaderCLI/Program.cs diff --git a/TwitchDownloaderCLI/Program.cs b/TwitchDownloaderCLI/Program.cs new file mode 100644 index 00000000..825e3ba2 --- /dev/null +++ b/TwitchDownloaderCLI/Program.cs @@ -0,0 +1,86 @@ +using CommandLine; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using CommandLine.Text; +using TwitchDownloaderCLI.Modes; +using TwitchDownloaderCLI.Modes.Arguments; +using TwitchDownloaderCLI.Tools; +using TwitchDownloaderCore.Tools; + +namespace TwitchDownloaderCLI +{ + internal static class Program + { + private static void Main(string[] args) + { + var preParsedArgs = PreParseArgs.Parse(args, Path.GetFileName(Environment.ProcessPath)); + + var parser = new Parser(config => + { + config.CaseInsensitiveEnumValues = true; + config.HelpWriter = TextWriter.Null; + }); + + var parserResult = parser.ParseArguments(preParsedArgs); + parserResult.WithNotParsed(errors => WriteHelpText(errors, parserResult, parser.Settings)); + + CoreLicensor.EnsureFilesExist(AppContext.BaseDirectory); + WriteApplicationBanner((ITwitchDownloaderArgs)parserResult.Value, args); + + parserResult + .WithParsed(DownloadVideo.Download) + .WithParsed(DownloadClip.Download) + .WithParsed(DownloadChat.Download) + .WithParsed(UpdateChat.Update) + .WithParsed(RenderChat.Render) + .WithParsed(FfmpegHandler.ParseArgs) + .WithParsed(CacheHandler.ParseArgs) + .WithParsed(MergeTS.Merge); + } + + private static void WriteHelpText(IEnumerable errors, ParserResult parserResult, ParserSettings parserSettings) + { + if (errors.FirstOrDefault()?.Tag == ErrorType.NoVerbSelectedError) + { + var processFileName = Path.GetFileName(Environment.ProcessPath); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + // Some Windows users try to double click the executable + Console.WriteLine("This is a command line tool. Please open a terminal and run \"{0} help\" from there for more information.{1}Press any key to close...", + processFileName, Environment.NewLine); + Console.ReadKey(); + } + else + { + Console.WriteLine("Usage: {0} [VERB] [OPTIONS]{1}Try \'{2} help\' for more information.", + processFileName, Environment.NewLine, processFileName); + } + } + else + { + Console.Error.WriteLine( + HelpText.AutoBuild(parserResult, builder => + { + builder.MaximumDisplayWidth = parserSettings.MaximumDisplayWidth; + builder.Copyright = CopyrightInfo.Default.ToString()!.Replace("\u00A9", "(c)"); + return builder; + })); + } + + Environment.Exit(1); + } + + private static void WriteApplicationBanner(ITwitchDownloaderArgs args, string[] argsArray) + { + if (args.ShowBanner == false || argsArray.Contains("--silent")) + { + return; + } + + Console.WriteLine($"{HeadingInfo.Default} {CopyrightInfo.Default.ToString()!.Replace("\u00A9", "(c)")}"); + } + } +} \ No newline at end of file From b9bfa639ce361cb90c25d4c4e5de787e927dc4c1 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:09:59 +0100 Subject: [PATCH 16/31] Delete TwitchDownloaderCore/TsMerger.cs --- TwitchDownloaderCore/TsMerger.cs | 132 ------------------------------- 1 file changed, 132 deletions(-) delete mode 100644 TwitchDownloaderCore/TsMerger.cs diff --git a/TwitchDownloaderCore/TsMerger.cs b/TwitchDownloaderCore/TsMerger.cs deleted file mode 100644 index fa29faf2..00000000 --- a/TwitchDownloaderCore/TsMerger.cs +++ /dev/null @@ -1,132 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using TwitchDownloaderCore.Options; -using TwitchDownloaderCore.Tools; - -namespace TwitchDownloaderCore -{ - public sealed class TsMerger - { - private readonly TsMergeOptions downloadOptions; - private readonly IProgress _progress; - - public TsMerger(TsMergeOptions tsMergeOptions, IProgress progress) - { - downloadOptions = tsMergeOptions; - _progress = progress; - } - - public async Task MergeAsync(CancellationToken cancellationToken) - { - string downloadFolder = ""; - - try - { - string InputList = downloadOptions.InputList; - List videoPartsList = System.IO.File.ReadLines(InputList).ToList(); - videoPartsList.RemoveAll(string.IsNullOrWhiteSpace); - - _progress.Report(new ProgressReport(ReportType.SameLineStatus, "Verifying Parts 0% [1/2]")); - - VerifyDownloadedParts(videoPartsList, downloadFolder, cancellationToken); - - _progress.Report(new ProgressReport() { ReportType = ReportType.NewLineStatus, Data = "Combining Parts 0% [2/2]" }); - - await CombineVideoParts(downloadFolder, videoPartsList, cancellationToken); - - _progress.Report(new ProgressReport(100)); - } - finally - { - } - } - - private void VerifyDownloadedParts(List videoParts, string downloadFolder, CancellationToken cancellationToken) - { - var failedParts = new List(); - var partCount = videoParts.Count; - var doneCount = 0; - - foreach (var part in videoParts) - { - if (!VerifyVideoPart(downloadFolder, part)) - { - failedParts.Add(part); - } - - doneCount++; - var percent = (int)(doneCount / (double)partCount * 100); - _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Verifying Parts {percent}% [1/2]")); - _progress.Report(new ProgressReport(percent)); - - cancellationToken.ThrowIfCancellationRequested(); - } - - if (failedParts.Count != 0) - { - if (failedParts.Count == videoParts.Count) - { - // Every video part returned corrupted, probably a false positive. - return; - } - - _progress.Report(new ProgressReport(ReportType.Log, $"The following parts appear to be invalid TS files: {string.Join(", ", failedParts)}")); - } - } - - private static bool VerifyVideoPart(string downloadFolder, string part) - { - const int TS_PACKET_LENGTH = 188; // MPEG TS packets are made of a header and a body: [ 4B ][ 184B ] - https://tsduck.io/download/docs/mpegts-introduction.pdf - - var partFile = Path.Combine(downloadFolder, part); - if (!File.Exists(partFile)) - { - return false; - } - - using var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.None); - var fileLength = fs.Length; - if (fileLength == 0 || fileLength % TS_PACKET_LENGTH != 0) - { - return false; - } - - return true; - } - - private async Task CombineVideoParts(string downloadFolder, List videoParts, CancellationToken cancellationToken) - { - DriveInfo outputDrive = DriveHelper.GetOutputDrive(downloadFolder); - string outputFile = Path.Combine(downloadFolder, downloadOptions.OutputFile); - - int partCount = videoParts.Count; - int doneCount = 0; - - await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.None); - foreach (var part in videoParts) - { - await DriveHelper.WaitForDrive(outputDrive, _progress, cancellationToken); - - string partFile = Path.Combine(downloadFolder, part); - if (File.Exists(partFile)) - { - await using (var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.None)) - { - await fs.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false); - } - } - - doneCount++; - int percent = (int)(doneCount / (double)partCount * 100); - _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Combining Parts {percent}% [2/2]")); - _progress.Report(new ProgressReport(percent)); - - cancellationToken.ThrowIfCancellationRequested(); - } - } - } -} From 84172c6d3106435c9f7886aa1c5dda8c9bd4057d Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:10:16 +0100 Subject: [PATCH 17/31] Add files via upload --- TwitchDownloaderCore/TsMerger.cs | 126 +++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 TwitchDownloaderCore/TsMerger.cs diff --git a/TwitchDownloaderCore/TsMerger.cs b/TwitchDownloaderCore/TsMerger.cs new file mode 100644 index 00000000..1303d253 --- /dev/null +++ b/TwitchDownloaderCore/TsMerger.cs @@ -0,0 +1,126 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using TwitchDownloaderCore.Options; +using TwitchDownloaderCore.Tools; + +namespace TwitchDownloaderCore +{ + public sealed class TsMerger + { + private readonly TsMergeOptions downloadOptions; + private readonly IProgress _progress; + + public TsMerger(TsMergeOptions tsMergeOptions, IProgress progress) + { + downloadOptions = tsMergeOptions; + _progress = progress; + } + + public async Task MergeAsync(CancellationToken cancellationToken) + { + string downloadFolder = ""; + + string InputList = downloadOptions.InputList; + List videoPartsList = System.IO.File.ReadLines(InputList).ToList(); + videoPartsList.RemoveAll(string.IsNullOrWhiteSpace); + + _progress.Report(new ProgressReport(ReportType.SameLineStatus, "Verifying Parts 0% [1/2]")); + + VerifyDownloadedParts(videoPartsList, downloadFolder, cancellationToken); + + _progress.Report(new ProgressReport() { ReportType = ReportType.NewLineStatus, Data = "Combining Parts 0% [2/2]" }); + + await CombineVideoParts(downloadFolder, videoPartsList, cancellationToken); + + _progress.Report(new ProgressReport(100)); + } + + private void VerifyDownloadedParts(List videoParts, string downloadFolder, CancellationToken cancellationToken) + { + var failedParts = new List(); + var partCount = videoParts.Count; + var doneCount = 0; + + foreach (var part in videoParts) + { + if (!VerifyVideoPart(downloadFolder, part)) + { + failedParts.Add(part); + } + + doneCount++; + var percent = (int)(doneCount / (double)partCount * 100); + _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Verifying Parts {percent}% [1/2]")); + _progress.Report(new ProgressReport(percent)); + + cancellationToken.ThrowIfCancellationRequested(); + } + + if (failedParts.Count != 0) + { + if (failedParts.Count == videoParts.Count) + { + // Every video part returned corrupted, probably a false positive. + return; + } + + _progress.Report(new ProgressReport(ReportType.Log, $"The following parts appear to be invalid TS files: {string.Join(", ", failedParts)}")); + } + } + + private static bool VerifyVideoPart(string downloadFolder, string part) + { + const int TS_PACKET_LENGTH = 188; // MPEG TS packets are made of a header and a body: [ 4B ][ 184B ] - https://tsduck.io/download/docs/mpegts-introduction.pdf + + var partFile = Path.Combine(downloadFolder, part); + if (!File.Exists(partFile)) + { + return false; + } + + using var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.None); + var fileLength = fs.Length; + if (fileLength == 0 || fileLength % TS_PACKET_LENGTH != 0) + { + return false; + } + + return true; + } + + private async Task CombineVideoParts(string downloadFolder, List videoParts, CancellationToken cancellationToken) + { + DriveInfo outputDrive = DriveHelper.GetOutputDrive(downloadFolder); + string outputFile = Path.Combine(downloadFolder, downloadOptions.OutputFile); + + int partCount = videoParts.Count; + int doneCount = 0; + + await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.None); + foreach (var part in videoParts) + { + await DriveHelper.WaitForDrive(outputDrive, _progress, cancellationToken); + + string partFile = Path.Combine(downloadFolder, part); + if (File.Exists(partFile)) + { + await using (var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.None)) + { + await fs.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false); + } + } + + doneCount++; + int percent = (int)(doneCount / (double)partCount * 100); + _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Combining Parts {percent}% [2/2]")); + _progress.Report(new ProgressReport(percent)); + + cancellationToken.ThrowIfCancellationRequested(); + } + } + } +} From 286f71cd485a1e6f335d435c0a458b69f8cac205 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:11:54 +0100 Subject: [PATCH 18/31] Delete TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs --- .../Modes/Arguments/TsMergeArgs.cs | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs diff --git a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs deleted file mode 100644 index c9c52737..00000000 --- a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs +++ /dev/null @@ -1,17 +0,0 @@ -using CommandLine; - -namespace TwitchDownloaderCLI.Modes.Arguments -{ - [Verb("tsmerge", HelpText = "Concatenates (not binary) .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file")] - public class TsMergeArgs : ITwitchDownloaderArgs - { - [Option('l', "inputlist", Required = true, HelpText = "Path to input list file in text format (one part per line).")] - public string InputList { get; set; } - - [Option('o', "output", Required = true, HelpText = "Path to output file.")] - public string OutputFile { get; set; } - - [Option("banner", Default = true, HelpText = "Displays a banner containing version and copyright information.")] - public bool? ShowBanner { get; set; } - } -} From e1553351f55d32086b813c0955542411d336c897 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:12:30 +0100 Subject: [PATCH 19/31] Add files via upload --- .../Modes/Arguments/TsMergeArgs.cs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs diff --git a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs new file mode 100644 index 00000000..35d93c4d --- /dev/null +++ b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs @@ -0,0 +1,17 @@ +using CommandLine; + +namespace TwitchDownloaderCLI.Modes.Arguments +{ + [Verb("tsmerge", HelpText = "Concatenates .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file")] + public class TsMergeArgs : ITwitchDownloaderArgs + { + [Option('l', "inputlist", Required = true, HelpText = "Path to input list file in text format (one part per line).")] + public string InputList { get; set; } + + [Option('o', "output", Required = true, HelpText = "Path to output file.")] + public string OutputFile { get; set; } + + [Option("banner", Default = true, HelpText = "Displays a banner containing version and copyright information.")] + public bool? ShowBanner { get; set; } + } +} From 8288281fcef576b59882390fcd7ff4db313dcabb Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:15:58 +0100 Subject: [PATCH 20/31] Delete TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs --- .../Modes/Arguments/TsMergeArgs.cs | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs diff --git a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs deleted file mode 100644 index 35d93c4d..00000000 --- a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs +++ /dev/null @@ -1,17 +0,0 @@ -using CommandLine; - -namespace TwitchDownloaderCLI.Modes.Arguments -{ - [Verb("tsmerge", HelpText = "Concatenates .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file")] - public class TsMergeArgs : ITwitchDownloaderArgs - { - [Option('l', "inputlist", Required = true, HelpText = "Path to input list file in text format (one part per line).")] - public string InputList { get; set; } - - [Option('o', "output", Required = true, HelpText = "Path to output file.")] - public string OutputFile { get; set; } - - [Option("banner", Default = true, HelpText = "Displays a banner containing version and copyright information.")] - public bool? ShowBanner { get; set; } - } -} From f8fbf3151d862bcd1b4e57cc5a85764e39b84daf Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:16:20 +0100 Subject: [PATCH 21/31] Add files via upload --- .../Modes/Arguments/TsMergeArgs.cs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs diff --git a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs new file mode 100644 index 00000000..1408c1df --- /dev/null +++ b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs @@ -0,0 +1,17 @@ +using CommandLine; + +namespace TwitchDownloaderCLI.Modes.Arguments +{ + [Verb("tsmerge", HelpText = "Concatenates .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file")] + public class TsMergeArgs : ITwitchDownloaderArgs + { + [Option('l', "inputlist", Required = true, HelpText = "Path to text file which contains the list of parts to concatenate (one path per line).")] + public string InputList { get; set; } + + [Option('o', "output", Required = true, HelpText = "Path to output file.")] + public string OutputFile { get; set; } + + [Option("banner", Default = true, HelpText = "Displays a banner containing version and copyright information.")] + public bool? ShowBanner { get; set; } + } +} From ee02787e35e69636b81704bdd407e7235448a1b0 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:37:22 +0100 Subject: [PATCH 22/31] Delete TwitchDownloaderCore/TsMerger.cs --- TwitchDownloaderCore/TsMerger.cs | 126 ------------------------------- 1 file changed, 126 deletions(-) delete mode 100644 TwitchDownloaderCore/TsMerger.cs diff --git a/TwitchDownloaderCore/TsMerger.cs b/TwitchDownloaderCore/TsMerger.cs deleted file mode 100644 index 1303d253..00000000 --- a/TwitchDownloaderCore/TsMerger.cs +++ /dev/null @@ -1,126 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using TwitchDownloaderCore.Options; -using TwitchDownloaderCore.Tools; - -namespace TwitchDownloaderCore -{ - public sealed class TsMerger - { - private readonly TsMergeOptions downloadOptions; - private readonly IProgress _progress; - - public TsMerger(TsMergeOptions tsMergeOptions, IProgress progress) - { - downloadOptions = tsMergeOptions; - _progress = progress; - } - - public async Task MergeAsync(CancellationToken cancellationToken) - { - string downloadFolder = ""; - - string InputList = downloadOptions.InputList; - List videoPartsList = System.IO.File.ReadLines(InputList).ToList(); - videoPartsList.RemoveAll(string.IsNullOrWhiteSpace); - - _progress.Report(new ProgressReport(ReportType.SameLineStatus, "Verifying Parts 0% [1/2]")); - - VerifyDownloadedParts(videoPartsList, downloadFolder, cancellationToken); - - _progress.Report(new ProgressReport() { ReportType = ReportType.NewLineStatus, Data = "Combining Parts 0% [2/2]" }); - - await CombineVideoParts(downloadFolder, videoPartsList, cancellationToken); - - _progress.Report(new ProgressReport(100)); - } - - private void VerifyDownloadedParts(List videoParts, string downloadFolder, CancellationToken cancellationToken) - { - var failedParts = new List(); - var partCount = videoParts.Count; - var doneCount = 0; - - foreach (var part in videoParts) - { - if (!VerifyVideoPart(downloadFolder, part)) - { - failedParts.Add(part); - } - - doneCount++; - var percent = (int)(doneCount / (double)partCount * 100); - _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Verifying Parts {percent}% [1/2]")); - _progress.Report(new ProgressReport(percent)); - - cancellationToken.ThrowIfCancellationRequested(); - } - - if (failedParts.Count != 0) - { - if (failedParts.Count == videoParts.Count) - { - // Every video part returned corrupted, probably a false positive. - return; - } - - _progress.Report(new ProgressReport(ReportType.Log, $"The following parts appear to be invalid TS files: {string.Join(", ", failedParts)}")); - } - } - - private static bool VerifyVideoPart(string downloadFolder, string part) - { - const int TS_PACKET_LENGTH = 188; // MPEG TS packets are made of a header and a body: [ 4B ][ 184B ] - https://tsduck.io/download/docs/mpegts-introduction.pdf - - var partFile = Path.Combine(downloadFolder, part); - if (!File.Exists(partFile)) - { - return false; - } - - using var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.None); - var fileLength = fs.Length; - if (fileLength == 0 || fileLength % TS_PACKET_LENGTH != 0) - { - return false; - } - - return true; - } - - private async Task CombineVideoParts(string downloadFolder, List videoParts, CancellationToken cancellationToken) - { - DriveInfo outputDrive = DriveHelper.GetOutputDrive(downloadFolder); - string outputFile = Path.Combine(downloadFolder, downloadOptions.OutputFile); - - int partCount = videoParts.Count; - int doneCount = 0; - - await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.None); - foreach (var part in videoParts) - { - await DriveHelper.WaitForDrive(outputDrive, _progress, cancellationToken); - - string partFile = Path.Combine(downloadFolder, part); - if (File.Exists(partFile)) - { - await using (var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.None)) - { - await fs.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false); - } - } - - doneCount++; - int percent = (int)(doneCount / (double)partCount * 100); - _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Combining Parts {percent}% [2/2]")); - _progress.Report(new ProgressReport(percent)); - - cancellationToken.ThrowIfCancellationRequested(); - } - } - } -} From 393e3c0da4383adb2f97e6197680d078516abbd5 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:37:40 +0100 Subject: [PATCH 23/31] Add files via upload --- TwitchDownloaderCore/TsMergeArgs.cs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 TwitchDownloaderCore/TsMergeArgs.cs diff --git a/TwitchDownloaderCore/TsMergeArgs.cs b/TwitchDownloaderCore/TsMergeArgs.cs new file mode 100644 index 00000000..1408c1df --- /dev/null +++ b/TwitchDownloaderCore/TsMergeArgs.cs @@ -0,0 +1,17 @@ +using CommandLine; + +namespace TwitchDownloaderCLI.Modes.Arguments +{ + [Verb("tsmerge", HelpText = "Concatenates .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file")] + public class TsMergeArgs : ITwitchDownloaderArgs + { + [Option('l', "inputlist", Required = true, HelpText = "Path to text file which contains the list of parts to concatenate (one path per line).")] + public string InputList { get; set; } + + [Option('o', "output", Required = true, HelpText = "Path to output file.")] + public string OutputFile { get; set; } + + [Option("banner", Default = true, HelpText = "Displays a banner containing version and copyright information.")] + public bool? ShowBanner { get; set; } + } +} From 00a7d4d59f1df4c5aa4029ec016b7028e7add346 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:38:30 +0100 Subject: [PATCH 24/31] Delete TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs --- .../Modes/Arguments/TsMergeArgs.cs | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs diff --git a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs deleted file mode 100644 index 1408c1df..00000000 --- a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs +++ /dev/null @@ -1,17 +0,0 @@ -using CommandLine; - -namespace TwitchDownloaderCLI.Modes.Arguments -{ - [Verb("tsmerge", HelpText = "Concatenates .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file")] - public class TsMergeArgs : ITwitchDownloaderArgs - { - [Option('l', "inputlist", Required = true, HelpText = "Path to text file which contains the list of parts to concatenate (one path per line).")] - public string InputList { get; set; } - - [Option('o', "output", Required = true, HelpText = "Path to output file.")] - public string OutputFile { get; set; } - - [Option("banner", Default = true, HelpText = "Displays a banner containing version and copyright information.")] - public bool? ShowBanner { get; set; } - } -} From 81ecdbe43d98c31ee1f16a12cd3d02bd7d392015 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:39:06 +0100 Subject: [PATCH 25/31] Add files via upload --- .../Modes/Arguments/TsMergeArgs.cs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs diff --git a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs new file mode 100644 index 00000000..8f553345 --- /dev/null +++ b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs @@ -0,0 +1,17 @@ +using CommandLine; + +namespace TwitchDownloaderCLI.Modes.Arguments +{ + [Verb("tsmerge", HelpText = "Concatenates .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file")] + public class TsMergeArgs : ITwitchDownloaderArgs + { + [Option('l', "inputlist", Required = true, HelpText = "Path to text file which contains the list of parts to concatenate.")] + public string InputList { get; set; } + + [Option('o', "output", Required = true, HelpText = "Path to output file.")] + public string OutputFile { get; set; } + + [Option("banner", Default = true, HelpText = "Displays a banner containing version and copyright information.")] + public bool? ShowBanner { get; set; } + } +} From abe5b40ad9dc816518c416b655ab95ae7de8697b Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:39:35 +0100 Subject: [PATCH 26/31] Delete TwitchDownloaderCore/TsMergeArgs.cs --- TwitchDownloaderCore/TsMergeArgs.cs | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 TwitchDownloaderCore/TsMergeArgs.cs diff --git a/TwitchDownloaderCore/TsMergeArgs.cs b/TwitchDownloaderCore/TsMergeArgs.cs deleted file mode 100644 index 1408c1df..00000000 --- a/TwitchDownloaderCore/TsMergeArgs.cs +++ /dev/null @@ -1,17 +0,0 @@ -using CommandLine; - -namespace TwitchDownloaderCLI.Modes.Arguments -{ - [Verb("tsmerge", HelpText = "Concatenates .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file")] - public class TsMergeArgs : ITwitchDownloaderArgs - { - [Option('l', "inputlist", Required = true, HelpText = "Path to text file which contains the list of parts to concatenate (one path per line).")] - public string InputList { get; set; } - - [Option('o', "output", Required = true, HelpText = "Path to output file.")] - public string OutputFile { get; set; } - - [Option("banner", Default = true, HelpText = "Displays a banner containing version and copyright information.")] - public bool? ShowBanner { get; set; } - } -} From 4c7206414456bc626a50a77092ad2c2629522165 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:39:55 +0100 Subject: [PATCH 27/31] Add files via upload --- TwitchDownloaderCore/TsMerger.cs | 119 +++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 TwitchDownloaderCore/TsMerger.cs diff --git a/TwitchDownloaderCore/TsMerger.cs b/TwitchDownloaderCore/TsMerger.cs new file mode 100644 index 00000000..540dd504 --- /dev/null +++ b/TwitchDownloaderCore/TsMerger.cs @@ -0,0 +1,119 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using TwitchDownloaderCore.Options; +using TwitchDownloaderCore.Tools; + +namespace TwitchDownloaderCore +{ + public sealed class TsMerger + { + private readonly TsMergeOptions downloadOptions; + private readonly IProgress _progress; + + public TsMerger(TsMergeOptions tsMergeOptions, IProgress progress) + { + downloadOptions = tsMergeOptions; + _progress = progress; + } + + public async Task MergeAsync(CancellationToken cancellationToken) + { + string InputList = downloadOptions.InputList; + List videoPartsList = System.IO.File.ReadLines(InputList).ToList(); + videoPartsList.RemoveAll(string.IsNullOrWhiteSpace); + + _progress.Report(new ProgressReport(ReportType.SameLineStatus, "Verifying Parts 0% [1/2]")); + + VerifyDownloadedParts(videoPartsList, cancellationToken); + + _progress.Report(new ProgressReport() { ReportType = ReportType.NewLineStatus, Data = "Combining Parts 0% [2/2]" }); + + await CombineVideoParts(videoPartsList, cancellationToken); + + _progress.Report(new ProgressReport(100)); + } + + private void VerifyDownloadedParts(List videoParts, CancellationToken cancellationToken) + { + var failedParts = new List(); + var partCount = videoParts.Count; + var doneCount = 0; + + foreach (var part in videoParts) + { + if (!VerifyVideoPart(part)) + { + failedParts.Add(part); + } + + doneCount++; + var percent = (int)(doneCount / (double)partCount * 100); + _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Verifying Parts {percent}% [1/2]")); + _progress.Report(new ProgressReport(percent)); + + cancellationToken.ThrowIfCancellationRequested(); + } + + if (failedParts.Count != 0) + { + if (failedParts.Count == videoParts.Count) + { + // Every video part returned corrupted, probably a false positive. + return; + } + + _progress.Report(new ProgressReport(ReportType.Log, $"The following parts appear to be invalid TS files: {string.Join(", ", failedParts)}")); + } + } + + private static bool VerifyVideoPart(string partFile) + { + const int TS_PACKET_LENGTH = 188; // MPEG TS packets are made of a header and a body: [ 4B ][ 184B ] - https://tsduck.io/download/docs/mpegts-introduction.pdf + + if (!File.Exists(partFile)) + { + return false; + } + + using var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.None); + var fileLength = fs.Length; + if (fileLength == 0 || fileLength % TS_PACKET_LENGTH != 0) + { + return false; + } + + return true; + } + + private async Task CombineVideoParts(List videoParts, CancellationToken cancellationToken) + { + DriveInfo outputDrive = DriveHelper.GetOutputDrive(downloadOptions.OutputFile); + string outputFile = downloadOptions.OutputFile; + + int partCount = videoParts.Count; + int doneCount = 0; + + await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.None); + foreach (var partFile in videoParts) + { + await DriveHelper.WaitForDrive(outputDrive, _progress, cancellationToken); + + await using (var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + await fs.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false); + } + + doneCount++; + int percent = (int)(doneCount / (double)partCount * 100); + _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Combining Parts {percent}% [2/2]")); + _progress.Report(new ProgressReport(percent)); + + cancellationToken.ThrowIfCancellationRequested(); + } + } + } +} From 60b3491724d2c7fd55878b4d7370994fe815f9b6 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:43:56 +0100 Subject: [PATCH 28/31] Delete TwitchDownloaderCLI/README.md --- TwitchDownloaderCLI/README.md | 420 ---------------------------------- 1 file changed, 420 deletions(-) delete mode 100644 TwitchDownloaderCLI/README.md diff --git a/TwitchDownloaderCLI/README.md b/TwitchDownloaderCLI/README.md deleted file mode 100644 index bbfb75a8..00000000 --- a/TwitchDownloaderCLI/README.md +++ /dev/null @@ -1,420 +0,0 @@ -# TwitchDownloaderCLI -A cross platform command line tool that can do the main functions of the GUI program, which can download VODs/Clips/Chats and render chats. -Also can concatenate/combine/merge Transport Stream files, either those parts downloaded with the CLI itself or from another source. - -- [TwitchDownloaderCLI](#twitchdownloadercli) - - [Arguments for mode videodownload](#arguments-for-mode-videodownload) - - [Arguments for mode clipdownload](#arguments-for-mode-clipdownload) - - [Arguments for mode chatdownload](#arguments-for-mode-chatdownload) - - [Arguments for mode chatupdate](#arguments-for-mode-chatupdate) - - [Arguments for mode chatrender](#arguments-for-mode-chatrender) - - [Arguments for mode ffmpeg](#arguments-for-mode-ffmpeg) - - [Arguments for mode cache](#arguments-for-mode-cache) - - [Arguments for mode tsmerge](#arguments-for-mode-tsmerge) - - [Example Commands](#example-commands) - - [Additional Notes](#additional-notes) - ---- - -## Arguments for mode videodownload -Downloads a stream VOD or highlight from Twitch - -**-u / --id (REQUIRED)** -The ID or URL of the VOD to download. - -**-o / --output (REQUIRED)** -File the program will output to. - -**-q / --quality** -The quality the program will attempt to download, for example "1080p60", if not found will download highest quality stream. - -**-b / --beginning** -Time in seconds to crop beginning. For example if I had a 10 second stream but only wanted the last 7 seconds of it I would use `-b 3` to skip the first 3 seconds. - -**-e / --ending** -Time in seconds to crop ending. For example if I had a 10 second stream but only wanted the first 4 seconds of it I would use `-e 4` to end on the 4th second. - -Extra example, if I wanted only seconds 3-6 in a 10 second stream I would do `-b 3 -e 6` - -**-t / --threads** -(Default: `4`) Number of download threads. - -**--bandwidth** -(Default: `-1`) The maximum bandwidth a thread will be allowed to use in kibibytes per second (KiB/s), or `-1` for no maximum. - -**--oauth** -OAuth access token to download subscriber only VODs. **DO NOT SHARE YOUR OAUTH TOKEN WITH ANYONE.** - -**--ffmpeg-path** -Path to FFmpeg executable. - -**--temp-path** -Path to temporary folder for cache. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - -## Arguments for mode clipdownload -Downloads a clip from Twitch - -**-u / --id (REQUIRED)** -The ID or URL of the Clip to download. - -**-o / --output (REQUIRED)** -File the program will output to. - -**-q / --quality** -The quality the program will attempt to download, for example "1080p60", if not found will download highest quality video. - -**--bandwidth** -(Default: `-1`) The maximum bandwidth the clip downloader is allowed to use in kibibytes per second (KiB/s), or `-1` for no maximum. - -**--encode-metadata** -(Default: `true`) Uses FFmpeg to add metadata to the clip output file. - -**--ffmpeg-path** -Path to FFmpeg executable. - -**--temp-path** -Path to temporary folder for cache. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - -## Arguments for mode chatdownload -Downloads the chat of a VOD, highlight, or clip - -**-u / --id (REQUIRED)** -The ID or URL of the VOD or clip to download. - -**-o / --output (REQUIRED)** -File the program will output to. File extension will be used to determine download type. Valid extensions are: `.json`, `.html`, and `.txt`. - -**--compression** -(Default: `None`) Compresses an output json chat file using a specified compression, usually resulting in 40-90% size reductions. Valid values are: `None`, `Gzip`. More formats will be supported in the future. - -**-b / --beginning** -Time in seconds to crop beginning. For example if I had a 10 second stream but only wanted the last 7 seconds of it I would use `-b 3` to skip the first 3 seconds. - -**-e / --ending** -Time in seconds to crop ending. For example if I had a 10 second stream but only wanted the first 4 seconds of it I would use `-e 4` to end on the 4th second. - -**-E / --embed-images** -(Default: `false`) Embed first party emotes, badges, and cheermotes into the download file for offline rendering. Useful for archival purposes, file size will be larger. - -**--bttv** -(Default: `true`) BTTV emote embedding. Requires `-E / --embed-images`. - -**--ffz** -(Default: `true`) FFZ emote embedding. Requires `-E / --embed-images`. - -**--stv** -(Default: `true`) 7TV emote embedding. Requires `-E / --embed-images`. - -**--timestamp-format** -(Default: `Relative`) Sets the timestamp format for .txt chat logs. Valid values are: `Utc`, `UtcFull`, `Relative`, and `None`. - -**--chat-connections** -(Default: `4`) The number of parallel downloads for chat. - -**--silent** -(Default: `false`) Suppresses progress console output. - -**--temp-path** -Path to temporary folder for cache. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - -## Arguments for mode chatupdate -Updates the embedded emotes, badges, bits, and crops of a chat download and/or converts a JSON chat to another format - -**-i / --input (REQUIRED)** -Path to input file. Valid extensions are: `.json`, `.json.gz`. - -**-o / --output (REQUIRED)** -Path to output file. File extension will be used to determine new chat type. Valid extensions are: `.json`, `.html`, and `.txt`. - -**-c / --compression** -(Default: `None`) Compresses an output json chat file using a specified compression, usually resulting in 40-90% size reductions. Valid values are: `None`, `Gzip`. More formats will be supported in the future. - -**-E / --embed-missing** -(Default: `false`) Embed missing emotes, badges, and cheermotes. Already embedded images will be untouched. - -**-R / --replace-embeds** -(Default: `false`) Replace all embedded emotes, badges, and cheermotes in the file. All embedded data will be overwritten! - -**b / --beginning** -(Default: `-1`) New time in seconds for chat beginning. Comments may be added but not removed. -1 = No crop. - -**-e / --ending** -(Default: `-1`) New time in seconds for chat beginning. Comments may be added but not removed. -1 = No crop. - -**--bttv** -(Default: `true`) Enable embedding BTTV emotes. - -**--ffz** -(Default: `true`) Enable embedding FFZ emotes. - -**--stv** -(Default: `true`) Enable embedding 7TV emotes. - -**--timestamp-format** -(Default: `Relative`) Sets the timestamp format for .txt chat logs. Valid values are: `Utc`, `Relative`, and `None`. - -**--temp-path** -Path to temporary folder for cache. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - -## Arguments for mode chatrender -Renders a chat JSON as a video - -**-i / --input (REQUIRED)** -The path to the `.json` or `.json.gz` chat file input. - -**-o / --output (REQUIRED)** -File the program will output to. - -**--background-color** -(Default: `#111111`) The render background color in the string format of `#RRGGBB` or `#AARRGGBB` in hexadecimal. - -**--alt-background-color** -(Default: `#191919`) The alternate message background color in the string format of `#RRGGBB` or `#AARRGGBB` in hexadecimal. Requires `--alternate-backgrounds`. - -**--message-color** -(Default: `#ffffff`) The message text color in the string format of `#RRGGBB` or `#AARRGGBB` in hexadecimal. - -**-w / --chat-width** -(Default: `350`) Width of chat render. - -**-h / --chat-height** -(Default: `600`) Height of chat render. - -**-b / --beginning** -(Default: `-1`) Time in seconds to crop the beginning of the render. - -**-e / --ending** -(Default: `-1`) Time in seconds to crop the ending of the render. - -**--bttv** -(Default: `true`) Enable BTTV emotes. - -**--ffz** -(Default: `true`) Enable FFZ emotes. - -**--stv** -(Default: `true`) Enable 7TV emotes. - -**--allow-unlisted-emotes** -(Default: `true`) Allow unlisted 7TV emotes in the render. - -**--sub-messages** -(Default: `true`) Enable sub / re-sub messages. - -**--badges** -(Default: `true`) Enable chat badges. - -**--outline** -(Default: `false`) Enable outline around chat messages. - -**--outline-size** -(Default: `4`) Size of outline if outline is enabled. - -**-f / --font** -(Default: `Inter Embedded`) Font to use. - -**--font-size** -(Default: `12`) Font size. - -**--message-fontstyle** -(Default: `normal`) Font style of message. Valid values are **normal**, **bold**, and **italic**. - -**--username-fontstyle** -(Default: `bold`) Font style of username. Valid values are **normal**, **bold**, and **italic**. - -**--timestamp** -(Default: `false`) Enables timestamps to left of messages, similar to VOD chat on Twitch. - -**--generate-mask** -(Default: `false`) Generates a mask file of the chat in addition to the rendered chat. - -**--sharpening** -(Default: `false`) Appends `-filter_complex "smartblur=lr=1:ls=-1.0"` to the `input-args`. Works best with `font-size` 24 or larger. - -**--framerate** -(Default: `30`) Framerate of the render. - -**--update-rate** -(Default: `0.2`) Time in seconds to update chat render output. - -**--input-args** -(Default: `-framerate {fps} -f rawvideo -analyzeduration {max_int} -probesize {max_int} -pix_fmt bgra -video_size {width}x{height} -i -`) Input arguments for FFmpeg chat render. - -**--output-args** -(Default: `-c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "{save_path}"`) Output arguments for FFmpeg chat render. - -**--ignore-users** -(Default: ` `) List of usernames to ignore when rendering, separated by commas. Not case-sensitive. - -**--ban-words** -(Default: ` `) List of words or phrases to ignore when rendering, separated by commas. Not case-sensitive. - -**--badge-filter** -(Default: `0`) Bitmask of types of Chat Badges to filter out. Add the numbers of the types of badges you want to filter. For example, to filter out Moderator and Broadcaster badges only enter the value of 6. - -Other = `1`, Broadcaster = `2`, Moderator = `4`, VIP = `8`, Subscriber = `16`, Predictions = `32`, NoAudioVisual = `64`, PrimeGaming = `128` - -**--dispersion** -(Default: `false`) In November 2022 a Twitch API change made chat messages download only in whole seconds. This option uses additional metadata to attempt to restore messages to when they were actually sent. This may result in a different comment order. Requires an update rate less than 1.0 for effective results. - -**--alternate-backgrounds** -(Default: `false`) Alternates the background color of every other chat message to help tell them apart. - -**--offline** -(Default: `false`) Render completely offline using only embedded emotes, badges, and bits from the input json. - -**--emoji-vendor** -(Default: `notocolor`) The emoji vendor used for rendering emojis. Valid values are: `twitter` / `twemoji`, `google` / `notocolor`, `none`. - -**--ffmpeg-path** -(Default: ` `) Path to FFmpeg executable. - -**--temp-path** -(Default: ` `) Path to temporary folder for cache. - -**--verbose-ffmpeg** -(Default: `false`) Prints every message from FFmpeg. - -**--skip-drive-waiting** -(Default: `false`) Do not wait for the output drive to transmit a ready signal before writing the next frame. Waiting is usually only necessary on low-end USB drives. Skipping can result in 1-5% render speed increases. - -**--scale-emote** -(Default: `1.0`) Number to scale emote images. - -**--scale-badge** -(Default: `1.0`) Number to scale badge images. - -**--scale-emoji** -(Default: `1.0`) Number to scale emoji images. - -**--scale-vertical** -(Default: `1.0`) Number to scale vertical padding. - -**--scale-side-padding** -(Default: `1.0`) Number to scale side padding. - -**--scale-section-height** -(Default: `1.0`) Number to scale section height of comments. - -**--scale-word-space** -(Default: `1.0`) Number to scale spacing between words. - -**--scale-emote-space** -(Default: `1.0`) Number to scale spacing between emotes. - -**--scale-highlight-stroke** -(Default: `1.0`) Number to scale highlight stroke size (sub messages). - -**--scale-highlight-indent** -(Default: `1.0`) Number to scale highlight indent size (sub messages). - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - - -## Arguments for mode ffmpeg -Manage standalone FFmpeg - -**-d / --download** -(Default: `false`) Downloads FFmpeg as a standalone file. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - -## Arguments for mode cache -Manage the working cache. - -**-c / --clear** -(Default: `false`) Clears the default cache folder. - -**--force-clear** -(Default: `false`) Clears the default cache folder, bypassing the confirmation prompt. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - -## Arguments for mode tsmerge -Concatenates .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file - -**-l / --inputlist (REQUIRED)** -Path to input list file in text format (one part per line). - -**-o / --output (REQUIRED)** -File the program will output to. - -**--banner** -(Default: `true`) Displays a banner containing version and copyright information. - ---- - -## Example Commands -Examples of typical use cases - -Download a VOD with defaults - - TwitchDownloaderCLI videodownload --id 612942303 -o video.mp4 - -Download a Clip with defaults - - TwitchDownloaderCLI clipdownload --id NurturingCalmHamburgerVoHiYo -o clip.mp4 - -Download a Chat JSON with embedded emotes/badges from Twitch and emotes from Bttv - - TwitchDownloaderCLI chatdownload --id 612942303 --embed-images --bttv=true --ffz=false --stv=false -o chat.json - -Download a Chat as plain text with timestamps - - TwitchDownloaderCLI chatdownload --id 612942303 --timestamp-format Relative -o chat.txt - -Add embeds to a chat file that was downloaded without embeds - - TwitchDownloaderCLI chatupdate -i chat.json -o chat_embedded.json --embed-missing - -Convert a JSON chat file to HTML - - TwitchDownloaderCLI chatupdate -i chat.json -o chat.html - -Render a chat with defaults - - TwitchDownloaderCLI chatrender -i chat.json -o chat.mp4 - -Render a chat with custom video settings and message outlines - - TwitchDownloaderCLI chatrender -i chat.json -h 1440 -w 720 --framerate 60 --outline -o chat.mp4 - -Render a chat with custom FFmpeg arguments - - TwitchDownloaderCLI chatrender -i chat.json --output-args='-c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "{save_path}"' -o chat.mp4 - -Concatenate several ts parts into another - - TwitchDownloaderCLI tsmerge -l list.txt -o output.ts - ---- - -## Additional Notes - -String arguments, such as output file, that contain spaces should be wrapped in either single quotes ' or double quotes " . - -Default true boolean flags must be assigned: `--default-true-flag=false`. Default false boolean flags should still be raised normally: `--default-false-flag` - -For Linux users, ensure both `fontconfig` and `libfontconfig1` are installed. `apt-get install fontconfig libfontconfig1` on Ubuntu. - -Some distros, like Linux Alpine, lack fonts for some languages (Arabic, Persian, Thai, etc.) If this is the case for you, install additional fonts families such as [Noto](https://fonts.google.com/noto/specimen/Noto+Sans) or check your distro's wiki page on fonts as it may have an install command for this specific scenario, such as the [Linux Alpine](https://wiki.alpinelinux.org/wiki/Fonts) font page. - -The list file for tsmerge can contain relative or absolute paths, but the format differs from Windows to Linux/UNIX/MacOS (like drive paths or "/" instead of "\" to separate directories). - -The concatenation made by tsmerge is not a raw (binary) concatenation, like `dd`, `cat` or `pv` would do on Linux, or `copy /b` on Windows. -Instead, the streams have to be partially recoded, to keep its structure valid and playable. Using other software may not achieve this result. From 263dff80b182dc77564d524cbbe370c72d8390a4 Mon Sep 17 00:00:00 2001 From: superbonaci Date: Sun, 12 Nov 2023 20:44:14 +0100 Subject: [PATCH 29/31] Add files via upload --- TwitchDownloaderCLI/README.md | 421 ++++++++++++++++++++++++++++++++++ 1 file changed, 421 insertions(+) create mode 100644 TwitchDownloaderCLI/README.md diff --git a/TwitchDownloaderCLI/README.md b/TwitchDownloaderCLI/README.md new file mode 100644 index 00000000..3f575c9f --- /dev/null +++ b/TwitchDownloaderCLI/README.md @@ -0,0 +1,421 @@ +# TwitchDownloaderCLI +A cross platform command line tool that can do the main functions of the GUI program, which can download VODs/Clips/Chats and render chats. +Also can concatenate/combine/merge Transport Stream files, either those parts downloaded with the CLI itself or from another source. + +- [TwitchDownloaderCLI](#twitchdownloadercli) + - [Arguments for mode videodownload](#arguments-for-mode-videodownload) + - [Arguments for mode clipdownload](#arguments-for-mode-clipdownload) + - [Arguments for mode chatdownload](#arguments-for-mode-chatdownload) + - [Arguments for mode chatupdate](#arguments-for-mode-chatupdate) + - [Arguments for mode chatrender](#arguments-for-mode-chatrender) + - [Arguments for mode ffmpeg](#arguments-for-mode-ffmpeg) + - [Arguments for mode cache](#arguments-for-mode-cache) + - [Arguments for mode tsmerge](#arguments-for-mode-tsmerge) + - [Example Commands](#example-commands) + - [Additional Notes](#additional-notes) + +--- + +## Arguments for mode videodownload +Downloads a stream VOD or highlight from Twitch + +**-u / --id (REQUIRED)** +The ID or URL of the VOD to download. + +**-o / --output (REQUIRED)** +File the program will output to. + +**-q / --quality** +The quality the program will attempt to download, for example "1080p60", if not found will download highest quality stream. + +**-b / --beginning** +Time in seconds to crop beginning. For example if I had a 10 second stream but only wanted the last 7 seconds of it I would use `-b 3` to skip the first 3 seconds. + +**-e / --ending** +Time in seconds to crop ending. For example if I had a 10 second stream but only wanted the first 4 seconds of it I would use `-e 4` to end on the 4th second. + +Extra example, if I wanted only seconds 3-6 in a 10 second stream I would do `-b 3 -e 6` + +**-t / --threads** +(Default: `4`) Number of download threads. + +**--bandwidth** +(Default: `-1`) The maximum bandwidth a thread will be allowed to use in kibibytes per second (KiB/s), or `-1` for no maximum. + +**--oauth** +OAuth access token to download subscriber only VODs. **DO NOT SHARE YOUR OAUTH TOKEN WITH ANYONE.** + +**--ffmpeg-path** +Path to FFmpeg executable. + +**--temp-path** +Path to temporary folder for cache. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +## Arguments for mode clipdownload +Downloads a clip from Twitch + +**-u / --id (REQUIRED)** +The ID or URL of the Clip to download. + +**-o / --output (REQUIRED)** +File the program will output to. + +**-q / --quality** +The quality the program will attempt to download, for example "1080p60", if not found will download highest quality video. + +**--bandwidth** +(Default: `-1`) The maximum bandwidth the clip downloader is allowed to use in kibibytes per second (KiB/s), or `-1` for no maximum. + +**--encode-metadata** +(Default: `true`) Uses FFmpeg to add metadata to the clip output file. + +**--ffmpeg-path** +Path to FFmpeg executable. + +**--temp-path** +Path to temporary folder for cache. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +## Arguments for mode chatdownload +Downloads the chat of a VOD, highlight, or clip + +**-u / --id (REQUIRED)** +The ID or URL of the VOD or clip to download. + +**-o / --output (REQUIRED)** +File the program will output to. File extension will be used to determine download type. Valid extensions are: `.json`, `.html`, and `.txt`. + +**--compression** +(Default: `None`) Compresses an output json chat file using a specified compression, usually resulting in 40-90% size reductions. Valid values are: `None`, `Gzip`. More formats will be supported in the future. + +**-b / --beginning** +Time in seconds to crop beginning. For example if I had a 10 second stream but only wanted the last 7 seconds of it I would use `-b 3` to skip the first 3 seconds. + +**-e / --ending** +Time in seconds to crop ending. For example if I had a 10 second stream but only wanted the first 4 seconds of it I would use `-e 4` to end on the 4th second. + +**-E / --embed-images** +(Default: `false`) Embed first party emotes, badges, and cheermotes into the download file for offline rendering. Useful for archival purposes, file size will be larger. + +**--bttv** +(Default: `true`) BTTV emote embedding. Requires `-E / --embed-images`. + +**--ffz** +(Default: `true`) FFZ emote embedding. Requires `-E / --embed-images`. + +**--stv** +(Default: `true`) 7TV emote embedding. Requires `-E / --embed-images`. + +**--timestamp-format** +(Default: `Relative`) Sets the timestamp format for .txt chat logs. Valid values are: `Utc`, `UtcFull`, `Relative`, and `None`. + +**--chat-connections** +(Default: `4`) The number of parallel downloads for chat. + +**--silent** +(Default: `false`) Suppresses progress console output. + +**--temp-path** +Path to temporary folder for cache. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +## Arguments for mode chatupdate +Updates the embedded emotes, badges, bits, and crops of a chat download and/or converts a JSON chat to another format + +**-i / --input (REQUIRED)** +Path to input file. Valid extensions are: `.json`, `.json.gz`. + +**-o / --output (REQUIRED)** +Path to output file. File extension will be used to determine new chat type. Valid extensions are: `.json`, `.html`, and `.txt`. + +**-c / --compression** +(Default: `None`) Compresses an output json chat file using a specified compression, usually resulting in 40-90% size reductions. Valid values are: `None`, `Gzip`. More formats will be supported in the future. + +**-E / --embed-missing** +(Default: `false`) Embed missing emotes, badges, and cheermotes. Already embedded images will be untouched. + +**-R / --replace-embeds** +(Default: `false`) Replace all embedded emotes, badges, and cheermotes in the file. All embedded data will be overwritten! + +**b / --beginning** +(Default: `-1`) New time in seconds for chat beginning. Comments may be added but not removed. -1 = No crop. + +**-e / --ending** +(Default: `-1`) New time in seconds for chat beginning. Comments may be added but not removed. -1 = No crop. + +**--bttv** +(Default: `true`) Enable embedding BTTV emotes. + +**--ffz** +(Default: `true`) Enable embedding FFZ emotes. + +**--stv** +(Default: `true`) Enable embedding 7TV emotes. + +**--timestamp-format** +(Default: `Relative`) Sets the timestamp format for .txt chat logs. Valid values are: `Utc`, `Relative`, and `None`. + +**--temp-path** +Path to temporary folder for cache. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +## Arguments for mode chatrender +Renders a chat JSON as a video + +**-i / --input (REQUIRED)** +The path to the `.json` or `.json.gz` chat file input. + +**-o / --output (REQUIRED)** +File the program will output to. + +**--background-color** +(Default: `#111111`) The render background color in the string format of `#RRGGBB` or `#AARRGGBB` in hexadecimal. + +**--alt-background-color** +(Default: `#191919`) The alternate message background color in the string format of `#RRGGBB` or `#AARRGGBB` in hexadecimal. Requires `--alternate-backgrounds`. + +**--message-color** +(Default: `#ffffff`) The message text color in the string format of `#RRGGBB` or `#AARRGGBB` in hexadecimal. + +**-w / --chat-width** +(Default: `350`) Width of chat render. + +**-h / --chat-height** +(Default: `600`) Height of chat render. + +**-b / --beginning** +(Default: `-1`) Time in seconds to crop the beginning of the render. + +**-e / --ending** +(Default: `-1`) Time in seconds to crop the ending of the render. + +**--bttv** +(Default: `true`) Enable BTTV emotes. + +**--ffz** +(Default: `true`) Enable FFZ emotes. + +**--stv** +(Default: `true`) Enable 7TV emotes. + +**--allow-unlisted-emotes** +(Default: `true`) Allow unlisted 7TV emotes in the render. + +**--sub-messages** +(Default: `true`) Enable sub / re-sub messages. + +**--badges** +(Default: `true`) Enable chat badges. + +**--outline** +(Default: `false`) Enable outline around chat messages. + +**--outline-size** +(Default: `4`) Size of outline if outline is enabled. + +**-f / --font** +(Default: `Inter Embedded`) Font to use. + +**--font-size** +(Default: `12`) Font size. + +**--message-fontstyle** +(Default: `normal`) Font style of message. Valid values are **normal**, **bold**, and **italic**. + +**--username-fontstyle** +(Default: `bold`) Font style of username. Valid values are **normal**, **bold**, and **italic**. + +**--timestamp** +(Default: `false`) Enables timestamps to left of messages, similar to VOD chat on Twitch. + +**--generate-mask** +(Default: `false`) Generates a mask file of the chat in addition to the rendered chat. + +**--sharpening** +(Default: `false`) Appends `-filter_complex "smartblur=lr=1:ls=-1.0"` to the `input-args`. Works best with `font-size` 24 or larger. + +**--framerate** +(Default: `30`) Framerate of the render. + +**--update-rate** +(Default: `0.2`) Time in seconds to update chat render output. + +**--input-args** +(Default: `-framerate {fps} -f rawvideo -analyzeduration {max_int} -probesize {max_int} -pix_fmt bgra -video_size {width}x{height} -i -`) Input arguments for FFmpeg chat render. + +**--output-args** +(Default: `-c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "{save_path}"`) Output arguments for FFmpeg chat render. + +**--ignore-users** +(Default: ` `) List of usernames to ignore when rendering, separated by commas. Not case-sensitive. + +**--ban-words** +(Default: ` `) List of words or phrases to ignore when rendering, separated by commas. Not case-sensitive. + +**--badge-filter** +(Default: `0`) Bitmask of types of Chat Badges to filter out. Add the numbers of the types of badges you want to filter. For example, to filter out Moderator and Broadcaster badges only enter the value of 6. + +Other = `1`, Broadcaster = `2`, Moderator = `4`, VIP = `8`, Subscriber = `16`, Predictions = `32`, NoAudioVisual = `64`, PrimeGaming = `128` + +**--dispersion** +(Default: `false`) In November 2022 a Twitch API change made chat messages download only in whole seconds. This option uses additional metadata to attempt to restore messages to when they were actually sent. This may result in a different comment order. Requires an update rate less than 1.0 for effective results. + +**--alternate-backgrounds** +(Default: `false`) Alternates the background color of every other chat message to help tell them apart. + +**--offline** +(Default: `false`) Render completely offline using only embedded emotes, badges, and bits from the input json. + +**--emoji-vendor** +(Default: `notocolor`) The emoji vendor used for rendering emojis. Valid values are: `twitter` / `twemoji`, `google` / `notocolor`, `none`. + +**--ffmpeg-path** +(Default: ` `) Path to FFmpeg executable. + +**--temp-path** +(Default: ` `) Path to temporary folder for cache. + +**--verbose-ffmpeg** +(Default: `false`) Prints every message from FFmpeg. + +**--skip-drive-waiting** +(Default: `false`) Do not wait for the output drive to transmit a ready signal before writing the next frame. Waiting is usually only necessary on low-end USB drives. Skipping can result in 1-5% render speed increases. + +**--scale-emote** +(Default: `1.0`) Number to scale emote images. + +**--scale-badge** +(Default: `1.0`) Number to scale badge images. + +**--scale-emoji** +(Default: `1.0`) Number to scale emoji images. + +**--scale-vertical** +(Default: `1.0`) Number to scale vertical padding. + +**--scale-side-padding** +(Default: `1.0`) Number to scale side padding. + +**--scale-section-height** +(Default: `1.0`) Number to scale section height of comments. + +**--scale-word-space** +(Default: `1.0`) Number to scale spacing between words. + +**--scale-emote-space** +(Default: `1.0`) Number to scale spacing between emotes. + +**--scale-highlight-stroke** +(Default: `1.0`) Number to scale highlight stroke size (sub messages). + +**--scale-highlight-indent** +(Default: `1.0`) Number to scale highlight indent size (sub messages). + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + + +## Arguments for mode ffmpeg +Manage standalone FFmpeg + +**-d / --download** +(Default: `false`) Downloads FFmpeg as a standalone file. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +## Arguments for mode cache +Manage the working cache. + +**-c / --clear** +(Default: `false`) Clears the default cache folder. + +**--force-clear** +(Default: `false`) Clears the default cache folder, bypassing the confirmation prompt. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +## Arguments for mode tsmerge +Concatenates .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file + +**-l / --inputlist (REQUIRED)** +Path to input list file in text format (one part per line). + +**-o / --output (REQUIRED)** +File the program will output to. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + +--- + +## Example Commands +Examples of typical use cases + +Download a VOD with defaults + + TwitchDownloaderCLI videodownload --id 612942303 -o video.mp4 + +Download a Clip with defaults + + TwitchDownloaderCLI clipdownload --id NurturingCalmHamburgerVoHiYo -o clip.mp4 + +Download a Chat JSON with embedded emotes/badges from Twitch and emotes from Bttv + + TwitchDownloaderCLI chatdownload --id 612942303 --embed-images --bttv=true --ffz=false --stv=false -o chat.json + +Download a Chat as plain text with timestamps + + TwitchDownloaderCLI chatdownload --id 612942303 --timestamp-format Relative -o chat.txt + +Add embeds to a chat file that was downloaded without embeds + + TwitchDownloaderCLI chatupdate -i chat.json -o chat_embedded.json --embed-missing + +Convert a JSON chat file to HTML + + TwitchDownloaderCLI chatupdate -i chat.json -o chat.html + +Render a chat with defaults + + TwitchDownloaderCLI chatrender -i chat.json -o chat.mp4 + +Render a chat with custom video settings and message outlines + + TwitchDownloaderCLI chatrender -i chat.json -h 1440 -w 720 --framerate 60 --outline -o chat.mp4 + +Render a chat with custom FFmpeg arguments + + TwitchDownloaderCLI chatrender -i chat.json --output-args='-c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "{save_path}"' -o chat.mp4 + +Concatenate several ts parts into another + + TwitchDownloaderCLI tsmerge -l list.txt -o output.ts + +--- + +## Additional Notes + +String arguments, such as output file, that contain spaces should be wrapped in either single quotes ' or double quotes " . + +Default true boolean flags must be assigned: `--default-true-flag=false`. Default false boolean flags should still be raised normally: `--default-false-flag` + +For Linux users, ensure both `fontconfig` and `libfontconfig1` are installed. `apt-get install fontconfig libfontconfig1` on Ubuntu. + +Some distros, like Linux Alpine, lack fonts for some languages (Arabic, Persian, Thai, etc.) If this is the case for you, install additional fonts families such as [Noto](https://fonts.google.com/noto/specimen/Noto+Sans) or check your distro's wiki page on fonts as it may have an install command for this specific scenario, such as the [Linux Alpine](https://wiki.alpinelinux.org/wiki/Fonts) font page. + +The list file for tsmerge can contain relative or absolute paths, but the path format differs from Windows to Linux/UNIX/MacOS (like volume paths or "/" instead of "\" to separate directories). +The list must be a text file and describe one ts path per line. + +The concatenation made by tsmerge is not a raw (binary) concatenation, like `dd`, `cat` or `pv` would do on Linux, or `copy /b` on Windows. +Instead, the streams have to be partially recoded, to keep its structure valid and playable. Using other software may not achieve this result. From a3134c9e99aafb8af2f437acf8b6dc79122947b9 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Tue, 23 Jan 2024 01:16:54 -0500 Subject: [PATCH 30/31] Cleanup code and fix Readme entry --- .../Modes/Arguments/TsMergeArgs.cs | 4 +- .../Modes/{MergeTS.cs => MergeTs.cs} | 6 +- TwitchDownloaderCLI/Program.cs | 2 +- TwitchDownloaderCLI/README.md | 14 ++--- .../Options/TsMergeOptions.cs | 2 +- TwitchDownloaderCore/TsMerger.cs | 63 ++++++++++++------- 6 files changed, 53 insertions(+), 38 deletions(-) rename TwitchDownloaderCLI/Modes/{MergeTS.cs => MergeTs.cs} (86%) diff --git a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs index 8f553345..1b7fffbe 100644 --- a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs +++ b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs @@ -2,10 +2,10 @@ namespace TwitchDownloaderCLI.Modes.Arguments { - [Verb("tsmerge", HelpText = "Concatenates .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file")] + [Verb("tsmerge", HelpText = "Concatenates multiple .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) files into a single file")] public class TsMergeArgs : ITwitchDownloaderArgs { - [Option('l', "inputlist", Required = true, HelpText = "Path to text file which contains the list of parts to concatenate.")] + [Option('i', "input", Required = true, HelpText = "Path a text file containing the absolute paths of the files to concatenate, separated by newlines. M3U/M3U8 is also supported.")] public string InputList { get; set; } [Option('o', "output", Required = true, HelpText = "Path to output file.")] diff --git a/TwitchDownloaderCLI/Modes/MergeTS.cs b/TwitchDownloaderCLI/Modes/MergeTs.cs similarity index 86% rename from TwitchDownloaderCLI/Modes/MergeTS.cs rename to TwitchDownloaderCLI/Modes/MergeTs.cs index 2d65a7a4..04edc5f8 100644 --- a/TwitchDownloaderCLI/Modes/MergeTS.cs +++ b/TwitchDownloaderCLI/Modes/MergeTs.cs @@ -1,15 +1,13 @@ using System; -using System.IO; using System.Threading; using TwitchDownloaderCLI.Modes.Arguments; using TwitchDownloaderCLI.Tools; using TwitchDownloaderCore; using TwitchDownloaderCore.Options; -using TwitchDownloaderCore.Tools; namespace TwitchDownloaderCLI.Modes { - internal static class MergeTS + internal static class MergeTs { internal static void Merge(TsMergeArgs inputOptions) { @@ -26,7 +24,7 @@ private static TsMergeOptions GetMergeOptions(TsMergeArgs inputOptions) TsMergeOptions mergeOptions = new() { OutputFile = inputOptions.OutputFile, - InputList = inputOptions.InputList + InputFile = inputOptions.InputList }; return mergeOptions; diff --git a/TwitchDownloaderCLI/Program.cs b/TwitchDownloaderCLI/Program.cs index 77d9ed1b..ed4d7b93 100644 --- a/TwitchDownloaderCLI/Program.cs +++ b/TwitchDownloaderCLI/Program.cs @@ -38,7 +38,7 @@ private static void Main(string[] args) .WithParsed(RenderChat.Render) .WithParsed(FfmpegHandler.ParseArgs) .WithParsed(CacheHandler.ParseArgs) - .WithParsed(MergeTS.Merge); + .WithParsed(MergeTs.Merge); } private static void WriteHelpText(IEnumerable errors, ParserResult parserResult, ParserSettings parserSettings) diff --git a/TwitchDownloaderCLI/README.md b/TwitchDownloaderCLI/README.md index 8662311e..ea57d57f 100644 --- a/TwitchDownloaderCLI/README.md +++ b/TwitchDownloaderCLI/README.md @@ -346,10 +346,10 @@ Other = `1`, Broadcaster = `2`, Moderator = `4`, VIP = `8`, Subscriber = `16`, P (Default: `true`) Displays a banner containing version and copyright information. ## Arguments for mode tsmerge -Concatenates .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) parts into another file +#### Concatenates multiple .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) files into a single file -**-l / --inputlist (REQUIRED)** -Path to input list file in text format (one part per line). +**-i / --input (REQUIRED)** +Path a text file containing the absolute paths of the files to concatenate, separated by newlines. M3U/M3U8 is also supported. **-o / --output (REQUIRED)** File the program will output to. @@ -408,6 +408,10 @@ Clear the default TwitchDownloader cache folder ./TwitchDownloaderCLI cache --clear +Concatenate several ts files into a single output file + + TwitchDownloaderCLI tsmerge -i list.txt -o output.ts + Print the available operations ./TwitchDownloaderCLI help @@ -416,10 +420,6 @@ Print the available options for the VOD downloader ./TwitchDownloaderCLI videodownload --help -Concatenate several ts parts into another - - TwitchDownloaderCLI tsmerge -l list.txt -o output.ts - --- ## Additional Notes diff --git a/TwitchDownloaderCore/Options/TsMergeOptions.cs b/TwitchDownloaderCore/Options/TsMergeOptions.cs index cac7df5f..8dd41e29 100644 --- a/TwitchDownloaderCore/Options/TsMergeOptions.cs +++ b/TwitchDownloaderCore/Options/TsMergeOptions.cs @@ -3,6 +3,6 @@ public class TsMergeOptions { public string OutputFile { get; set; } - public string InputList { get; set; } + public string InputFile { get; set; } } } diff --git a/TwitchDownloaderCore/TsMerger.cs b/TwitchDownloaderCore/TsMerger.cs index 540dd504..b6193f10 100644 --- a/TwitchDownloaderCore/TsMerger.cs +++ b/TwitchDownloaderCore/TsMerger.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Threading; using System.Threading.Tasks; using TwitchDownloaderCore.Options; @@ -11,41 +10,59 @@ namespace TwitchDownloaderCore { public sealed class TsMerger { - private readonly TsMergeOptions downloadOptions; + private readonly TsMergeOptions mergeOptions; private readonly IProgress _progress; public TsMerger(TsMergeOptions tsMergeOptions, IProgress progress) { - downloadOptions = tsMergeOptions; + mergeOptions = tsMergeOptions; _progress = progress; } public async Task MergeAsync(CancellationToken cancellationToken) { - string InputList = downloadOptions.InputList; - List videoPartsList = System.IO.File.ReadLines(InputList).ToList(); - videoPartsList.RemoveAll(string.IsNullOrWhiteSpace); + if (!File.Exists(mergeOptions.InputFile)) + { + throw new FileNotFoundException("Input file does not exist"); + } + + var fileList = new List(); + await using (var fs = File.Open(mergeOptions.InputFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + { + using var sr = new StreamReader(fs); + while (await sr.ReadLineAsync() is { } line) + { + if (string.IsNullOrWhiteSpace(line)) + continue; + + if (line.StartsWith('#')) + continue; + + fileList.Add(line); + } + } _progress.Report(new ProgressReport(ReportType.SameLineStatus, "Verifying Parts 0% [1/2]")); - VerifyDownloadedParts(videoPartsList, cancellationToken); + await VerifyVideoParts(fileList, cancellationToken); _progress.Report(new ProgressReport() { ReportType = ReportType.NewLineStatus, Data = "Combining Parts 0% [2/2]" }); - await CombineVideoParts(videoPartsList, cancellationToken); + await CombineVideoParts(fileList, cancellationToken); _progress.Report(new ProgressReport(100)); } - private void VerifyDownloadedParts(List videoParts, CancellationToken cancellationToken) + private async Task VerifyVideoParts(IReadOnlyCollection fileList, CancellationToken cancellationToken) { var failedParts = new List(); - var partCount = videoParts.Count; + var partCount = fileList.Count; var doneCount = 0; - foreach (var part in videoParts) + foreach (var part in fileList) { - if (!VerifyVideoPart(part)) + var isValidTs = await VerifyVideoPart(part); + if (!isValidTs) { failedParts.Add(part); } @@ -60,26 +77,26 @@ private void VerifyDownloadedParts(List videoParts, CancellationToken ca if (failedParts.Count != 0) { - if (failedParts.Count == videoParts.Count) + if (failedParts.Count == fileList.Count) { // Every video part returned corrupted, probably a false positive. return; } - _progress.Report(new ProgressReport(ReportType.Log, $"The following parts appear to be invalid TS files: {string.Join(", ", failedParts)}")); + _progress.Report(new ProgressReport(ReportType.Log, $"The following TS files appear to be invalid or corrupted: {string.Join(", ", failedParts)}")); } } - private static bool VerifyVideoPart(string partFile) + private static async Task VerifyVideoPart(string filePath) { const int TS_PACKET_LENGTH = 188; // MPEG TS packets are made of a header and a body: [ 4B ][ 184B ] - https://tsduck.io/download/docs/mpegts-introduction.pdf - if (!File.Exists(partFile)) + if (!File.Exists(filePath)) { return false; } - using var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.None); + await using var fs = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); var fileLength = fs.Length; if (fileLength == 0 || fileLength % TS_PACKET_LENGTH != 0) { @@ -89,16 +106,16 @@ private static bool VerifyVideoPart(string partFile) return true; } - private async Task CombineVideoParts(List videoParts, CancellationToken cancellationToken) + private async Task CombineVideoParts(IReadOnlyCollection fileList, CancellationToken cancellationToken) { - DriveInfo outputDrive = DriveHelper.GetOutputDrive(downloadOptions.OutputFile); - string outputFile = downloadOptions.OutputFile; + DriveInfo outputDrive = DriveHelper.GetOutputDrive(mergeOptions.OutputFile); + string outputFile = mergeOptions.OutputFile; - int partCount = videoParts.Count; + int partCount = fileList.Count; int doneCount = 0; - await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.None); - foreach (var partFile in videoParts) + await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.Read); + foreach (var partFile in fileList) { await DriveHelper.WaitForDrive(outputDrive, _progress, cancellationToken); From 98a37711da794756c6989e1ea042e68fe906a3d3 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Sun, 25 Feb 2024 14:55:44 -0500 Subject: [PATCH 31/31] Cleanup --- TwitchDownloaderCLI/Modes/MergeTs.cs | 2 ++ TwitchDownloaderCLI/README.md | 7 ++----- TwitchDownloaderCore/TsMerger.cs | 16 +++++++++++----- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/TwitchDownloaderCLI/Modes/MergeTs.cs b/TwitchDownloaderCLI/Modes/MergeTs.cs index 04edc5f8..13c41c65 100644 --- a/TwitchDownloaderCLI/Modes/MergeTs.cs +++ b/TwitchDownloaderCLI/Modes/MergeTs.cs @@ -11,6 +11,8 @@ internal static class MergeTs { internal static void Merge(TsMergeArgs inputOptions) { + Console.WriteLine("[INFO] The TS merger is experimental and is subject to change without notice in future releases."); + Progress progress = new(); progress.ProgressChanged += ProgressHandler.Progress_ProgressChanged; diff --git a/TwitchDownloaderCLI/README.md b/TwitchDownloaderCLI/README.md index ea57d57f..5d4b5ebe 100644 --- a/TwitchDownloaderCLI/README.md +++ b/TwitchDownloaderCLI/README.md @@ -434,8 +434,5 @@ For Linux users, ensure both `fontconfig` and `libfontconfig1` are installed. `a Some distros, like Linux Alpine, lack fonts for some languages (Arabic, Persian, Thai, etc.) If this is the case for you, install additional fonts families such as [Noto](https://fonts.google.com/noto/specimen/Noto+Sans) or check your distro's wiki page on fonts as it may have an install command for this specific scenario, such as the [Linux Alpine](https://wiki.alpinelinux.org/wiki/Fonts) font page. -The list file for tsmerge can contain relative or absolute paths, but the path format differs from Windows to Linux/UNIX/MacOS (like volume paths or "/" instead of "\" to separate directories). -The list must be a text file and describe one ts path per line. - -The concatenation made by tsmerge is not a raw (binary) concatenation, like `dd`, `cat` or `pv` would do on Linux, or `copy /b` on Windows. -Instead, the streams have to be partially recoded, to keep its structure valid and playable. Using other software may not achieve this result. +The list file for `tsmerge` may contain relative or absolute paths, with one path per line. +Alternatively, the list file may also be an M3U8 playlist file. \ No newline at end of file diff --git a/TwitchDownloaderCore/TsMerger.cs b/TwitchDownloaderCore/TsMerger.cs index b6193f10..e3e54128 100644 --- a/TwitchDownloaderCore/TsMerger.cs +++ b/TwitchDownloaderCore/TsMerger.cs @@ -26,17 +26,23 @@ public async Task MergeAsync(CancellationToken cancellationToken) throw new FileNotFoundException("Input file does not exist"); } + var isM3U8 = false; var fileList = new List(); await using (var fs = File.Open(mergeOptions.InputFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { using var sr = new StreamReader(fs); while (await sr.ReadLineAsync() is { } line) { - if (string.IsNullOrWhiteSpace(line)) - continue; + if (string.IsNullOrWhiteSpace(line)) continue; - if (line.StartsWith('#')) - continue; + if (isM3U8) + { + if (line.StartsWith('#')) continue; + } + else + { + if (line.StartsWith("#EXTM3U")) isM3U8 = true; + } fileList.Add(line); } @@ -83,7 +89,7 @@ private async Task VerifyVideoParts(IReadOnlyCollection fileList, Cancel return; } - _progress.Report(new ProgressReport(ReportType.Log, $"The following TS files appear to be invalid or corrupted: {string.Join(", ", failedParts)}")); + _progress.Report(new ProgressReport(ReportType.Log, $"The following TS files are invalid or corrupted: {string.Join(", ", failedParts)}")); } }