diff --git a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs new file mode 100644 index 00000000..1b7fffbe --- /dev/null +++ b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs @@ -0,0 +1,17 @@ +using CommandLine; + +namespace TwitchDownloaderCLI.Modes.Arguments +{ + [Verb("tsmerge", HelpText = "Concatenates multiple .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) files into a single file")] + public class TsMergeArgs : ITwitchDownloaderArgs + { + [Option('i', "input", Required = true, HelpText = "Path a text file containing the absolute paths of the files to concatenate, separated by newlines. M3U/M3U8 is also supported.")] + public string InputList { get; set; } + + [Option('o', "output", Required = true, HelpText = "Path to output file.")] + public string OutputFile { get; set; } + + [Option("banner", Default = true, HelpText = "Displays a banner containing version and copyright information.")] + public bool? ShowBanner { get; set; } + } +} diff --git a/TwitchDownloaderCLI/Modes/MergeTs.cs b/TwitchDownloaderCLI/Modes/MergeTs.cs new file mode 100644 index 00000000..13c41c65 --- /dev/null +++ b/TwitchDownloaderCLI/Modes/MergeTs.cs @@ -0,0 +1,35 @@ +using System; +using System.Threading; +using TwitchDownloaderCLI.Modes.Arguments; +using TwitchDownloaderCLI.Tools; +using TwitchDownloaderCore; +using TwitchDownloaderCore.Options; + +namespace TwitchDownloaderCLI.Modes +{ + internal static class MergeTs + { + internal static void Merge(TsMergeArgs inputOptions) + { + Console.WriteLine("[INFO] The TS merger is experimental and is subject to change without notice in future releases."); + + Progress progress = new(); + progress.ProgressChanged += ProgressHandler.Progress_ProgressChanged; + + var mergeOptions = GetMergeOptions(inputOptions); + TsMerger tsMerger = new(mergeOptions, progress); + tsMerger.MergeAsync(new CancellationToken()).Wait(); + } + + private static TsMergeOptions GetMergeOptions(TsMergeArgs inputOptions) + { + TsMergeOptions mergeOptions = new() + { + OutputFile = inputOptions.OutputFile, + InputFile = inputOptions.InputList + }; + + return mergeOptions; + } + } +} diff --git a/TwitchDownloaderCLI/Program.cs b/TwitchDownloaderCLI/Program.cs index 23121d65..4e40a757 100644 --- a/TwitchDownloaderCLI/Program.cs +++ b/TwitchDownloaderCLI/Program.cs @@ -24,7 +24,7 @@ private static void Main(string[] args) config.HelpWriter = null; // Use null instead of TextWriter.Null due to how CommandLine works internally }); - var parserResult = parser.ParseArguments(preParsedArgs); + var parserResult = parser.ParseArguments(preParsedArgs); parserResult.WithNotParsed(errors => WriteHelpText(errors, parserResult, parser.Settings)); CoreLicensor.EnsureFilesExist(AppContext.BaseDirectory); @@ -37,7 +37,8 @@ private static void Main(string[] args) .WithParsed(UpdateChat.Update) .WithParsed(RenderChat.Render) .WithParsed(FfmpegHandler.ParseArgs) - .WithParsed(CacheHandler.ParseArgs); + .WithParsed(CacheHandler.ParseArgs) + .WithParsed(MergeTs.Merge); } private static void WriteHelpText(IEnumerable errors, ParserResult parserResult, ParserSettings parserSettings) diff --git a/TwitchDownloaderCLI/README.md b/TwitchDownloaderCLI/README.md index 2a23c703..5d4b5ebe 100644 --- a/TwitchDownloaderCLI/README.md +++ b/TwitchDownloaderCLI/README.md @@ -1,5 +1,6 @@ # TwitchDownloaderCLI A cross platform command line tool that can do the main functions of the GUI program, which can download VODs/Clips/Chats and render chats. +Also can concatenate/combine/merge Transport Stream files, either those parts downloaded with the CLI itself or from another source. - [TwitchDownloaderCLI](#twitchdownloadercli) - [Arguments for mode videodownload](#arguments-for-mode-videodownload) @@ -9,6 +10,7 @@ A cross platform command line tool that can do the main functions of the GUI pro - [Arguments for mode chatrender](#arguments-for-mode-chatrender) - [Arguments for mode ffmpeg](#arguments-for-mode-ffmpeg) - [Arguments for mode cache](#arguments-for-mode-cache) + - [Arguments for mode tsmerge](#arguments-for-mode-tsmerge) - [Example Commands](#example-commands) - [Additional Notes](#additional-notes) @@ -343,6 +345,18 @@ Other = `1`, Broadcaster = `2`, Moderator = `4`, VIP = `8`, Subscriber = `16`, P **--banner** (Default: `true`) Displays a banner containing version and copyright information. +## Arguments for mode tsmerge +#### Concatenates multiple .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) files into a single file + +**-i / --input (REQUIRED)** +Path a text file containing the absolute paths of the files to concatenate, separated by newlines. M3U/M3U8 is also supported. + +**-o / --output (REQUIRED)** +File the program will output to. + +**--banner** +(Default: `true`) Displays a banner containing version and copyright information. + --- ## Example Commands @@ -394,6 +408,10 @@ Clear the default TwitchDownloader cache folder ./TwitchDownloaderCLI cache --clear +Concatenate several ts files into a single output file + + TwitchDownloaderCLI tsmerge -i list.txt -o output.ts + Print the available operations ./TwitchDownloaderCLI help @@ -415,3 +433,6 @@ Default true boolean flags must be assigned: `--default-true-flag=false`. Defaul For Linux users, ensure both `fontconfig` and `libfontconfig1` are installed. `apt-get install fontconfig libfontconfig1` on Ubuntu. Some distros, like Linux Alpine, lack fonts for some languages (Arabic, Persian, Thai, etc.) If this is the case for you, install additional fonts families such as [Noto](https://fonts.google.com/noto/specimen/Noto+Sans) or check your distro's wiki page on fonts as it may have an install command for this specific scenario, such as the [Linux Alpine](https://wiki.alpinelinux.org/wiki/Fonts) font page. + +The list file for `tsmerge` may contain relative or absolute paths, with one path per line. +Alternatively, the list file may also be an M3U8 playlist file. \ No newline at end of file diff --git a/TwitchDownloaderCore/Options/TsMergeOptions.cs b/TwitchDownloaderCore/Options/TsMergeOptions.cs new file mode 100644 index 00000000..8dd41e29 --- /dev/null +++ b/TwitchDownloaderCore/Options/TsMergeOptions.cs @@ -0,0 +1,8 @@ +namespace TwitchDownloaderCore.Options +{ + public class TsMergeOptions + { + public string OutputFile { get; set; } + public string InputFile { get; set; } + } +} diff --git a/TwitchDownloaderCore/TsMerger.cs b/TwitchDownloaderCore/TsMerger.cs new file mode 100644 index 00000000..e3e54128 --- /dev/null +++ b/TwitchDownloaderCore/TsMerger.cs @@ -0,0 +1,142 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using TwitchDownloaderCore.Options; +using TwitchDownloaderCore.Tools; + +namespace TwitchDownloaderCore +{ + public sealed class TsMerger + { + private readonly TsMergeOptions mergeOptions; + private readonly IProgress _progress; + + public TsMerger(TsMergeOptions tsMergeOptions, IProgress progress) + { + mergeOptions = tsMergeOptions; + _progress = progress; + } + + public async Task MergeAsync(CancellationToken cancellationToken) + { + if (!File.Exists(mergeOptions.InputFile)) + { + throw new FileNotFoundException("Input file does not exist"); + } + + var isM3U8 = false; + var fileList = new List(); + await using (var fs = File.Open(mergeOptions.InputFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + { + using var sr = new StreamReader(fs); + while (await sr.ReadLineAsync() is { } line) + { + if (string.IsNullOrWhiteSpace(line)) continue; + + if (isM3U8) + { + if (line.StartsWith('#')) continue; + } + else + { + if (line.StartsWith("#EXTM3U")) isM3U8 = true; + } + + fileList.Add(line); + } + } + + _progress.Report(new ProgressReport(ReportType.SameLineStatus, "Verifying Parts 0% [1/2]")); + + await VerifyVideoParts(fileList, cancellationToken); + + _progress.Report(new ProgressReport() { ReportType = ReportType.NewLineStatus, Data = "Combining Parts 0% [2/2]" }); + + await CombineVideoParts(fileList, cancellationToken); + + _progress.Report(new ProgressReport(100)); + } + + private async Task VerifyVideoParts(IReadOnlyCollection fileList, CancellationToken cancellationToken) + { + var failedParts = new List(); + var partCount = fileList.Count; + var doneCount = 0; + + foreach (var part in fileList) + { + var isValidTs = await VerifyVideoPart(part); + if (!isValidTs) + { + failedParts.Add(part); + } + + doneCount++; + var percent = (int)(doneCount / (double)partCount * 100); + _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Verifying Parts {percent}% [1/2]")); + _progress.Report(new ProgressReport(percent)); + + cancellationToken.ThrowIfCancellationRequested(); + } + + if (failedParts.Count != 0) + { + if (failedParts.Count == fileList.Count) + { + // Every video part returned corrupted, probably a false positive. + return; + } + + _progress.Report(new ProgressReport(ReportType.Log, $"The following TS files are invalid or corrupted: {string.Join(", ", failedParts)}")); + } + } + + private static async Task VerifyVideoPart(string filePath) + { + const int TS_PACKET_LENGTH = 188; // MPEG TS packets are made of a header and a body: [ 4B ][ 184B ] - https://tsduck.io/download/docs/mpegts-introduction.pdf + + if (!File.Exists(filePath)) + { + return false; + } + + await using var fs = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); + var fileLength = fs.Length; + if (fileLength == 0 || fileLength % TS_PACKET_LENGTH != 0) + { + return false; + } + + return true; + } + + private async Task CombineVideoParts(IReadOnlyCollection fileList, CancellationToken cancellationToken) + { + DriveInfo outputDrive = DriveHelper.GetOutputDrive(mergeOptions.OutputFile); + string outputFile = mergeOptions.OutputFile; + + int partCount = fileList.Count; + int doneCount = 0; + + await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.Read); + foreach (var partFile in fileList) + { + await DriveHelper.WaitForDrive(outputDrive, _progress, cancellationToken); + + await using (var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + await fs.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false); + } + + doneCount++; + int percent = (int)(doneCount / (double)partCount * 100); + _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Combining Parts {percent}% [2/2]")); + _progress.Report(new ProgressReport(percent)); + + cancellationToken.ThrowIfCancellationRequested(); + } + } + } +}