Fixes #848 add CLI mode to manually concatenate ts parts

TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs

@@ -0,0 +1,17 @@
+using CommandLine;
+
+namespace TwitchDownloaderCLI.Modes.Arguments
+{
+    [Verb("tsmerge", HelpText = "Concatenates multiple .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) files into a single file")]
+    public class TsMergeArgs : ITwitchDownloaderArgs
+    {
+        [Option('i', "input", Required = true, HelpText = "Path a text file containing the absolute paths of the files to concatenate, separated by newlines. M3U/M3U8 is also supported.")]
+        public string InputList { get; set; }
+
+        [Option('o', "output", Required = true, HelpText = "Path to output file.")]
+        public string OutputFile { get; set; }
+
+        [Option("banner", Default = true, HelpText = "Displays a banner containing version and copyright information.")]
+        public bool? ShowBanner { get; set; }
+    }
+}

TwitchDownloaderCLI/Modes/MergeTs.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Threading;
+using TwitchDownloaderCLI.Modes.Arguments;
+using TwitchDownloaderCLI.Tools;
+using TwitchDownloaderCore;
+using TwitchDownloaderCore.Options;
+
+namespace TwitchDownloaderCLI.Modes
+{
+    internal static class MergeTs
+    {
+        internal static void Merge(TsMergeArgs inputOptions)
+        {
+            Console.WriteLine("[INFO] The TS merger is experimental and is subject to change without notice in future releases.");
+
+            Progress<ProgressReport> progress = new();
+            progress.ProgressChanged += ProgressHandler.Progress_ProgressChanged;
+
+            var mergeOptions = GetMergeOptions(inputOptions);
+            TsMerger tsMerger = new(mergeOptions, progress);
+            tsMerger.MergeAsync(new CancellationToken()).Wait();
+        }
+
+        private static TsMergeOptions GetMergeOptions(TsMergeArgs inputOptions)
+        {
+            TsMergeOptions mergeOptions = new()
+            {
+                OutputFile = inputOptions.OutputFile,
+                InputFile = inputOptions.InputList
+            };
+
+            return mergeOptions;
+        }
+    }
+}

TwitchDownloaderCLI/Program.cs

@@ -24,7 +24,7 @@ private static void Main(string[] args)
                 config.HelpWriter = null; // Use null instead of TextWriter.Null due to how CommandLine works internally
             });
 
-            var parserResult = parser.ParseArguments<VideoDownloadArgs, ClipDownloadArgs, ChatDownloadArgs, ChatUpdateArgs, ChatRenderArgs, FfmpegArgs, CacheArgs>(preParsedArgs);
+            var parserResult = parser.ParseArguments<VideoDownloadArgs, ClipDownloadArgs, ChatDownloadArgs, ChatUpdateArgs, ChatRenderArgs, FfmpegArgs, CacheArgs, TsMergeArgs>(preParsedArgs);
             parserResult.WithNotParsed(errors => WriteHelpText(errors, parserResult, parser.Settings));
 
             CoreLicensor.EnsureFilesExist(AppContext.BaseDirectory);
@@ -37,7 +37,8 @@ private static void Main(string[] args)
                 .WithParsed<ChatUpdateArgs>(UpdateChat.Update)
                 .WithParsed<ChatRenderArgs>(RenderChat.Render)
                 .WithParsed<FfmpegArgs>(FfmpegHandler.ParseArgs)
-                .WithParsed<CacheArgs>(CacheHandler.ParseArgs);
+                .WithParsed<CacheArgs>(CacheHandler.ParseArgs)
+                .WithParsed<TsMergeArgs>(MergeTs.Merge);
         }
 
         private static void WriteHelpText(IEnumerable<Error> errors, ParserResult<object> parserResult, ParserSettings parserSettings)

TwitchDownloaderCLI/README.md

@@ -1,5 +1,6 @@
 # TwitchDownloaderCLI
 A cross platform command line tool that can do the main functions of the GUI program, which can download VODs/Clips/Chats and render chats.
+Also can concatenate/combine/merge Transport Stream files, either those parts downloaded with the CLI itself or from another source.
 
 - [TwitchDownloaderCLI](#twitchdownloadercli)
   - [Arguments for mode videodownload](#arguments-for-mode-videodownload)
@@ -9,6 +10,7 @@ A cross platform command line tool that can do the main functions of the GUI pro
   - [Arguments for mode chatrender](#arguments-for-mode-chatrender)
   - [Arguments for mode ffmpeg](#arguments-for-mode-ffmpeg)
   - [Arguments for mode cache](#arguments-for-mode-cache)
+  - [Arguments for mode tsmerge](#arguments-for-mode-tsmerge)
   - [Example Commands](#example-commands)
   - [Additional Notes](#additional-notes)
 
@@ -343,6 +345,18 @@ Other = `1`, Broadcaster = `2`, Moderator = `4`, VIP = `8`, Subscriber = `16`, P
 **--banner**
 (Default: `true`) Displays a banner containing version and copyright information.
 
+## Arguments for mode tsmerge
+#### Concatenates multiple .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) files into a single file
+
+**-i / --input (REQUIRED)**
+Path a text file containing the absolute paths of the files to concatenate, separated by newlines. M3U/M3U8 is also supported.
+
+**-o / --output (REQUIRED)**
+File the program will output to.
+
+**--banner**
+(Default: `true`) Displays a banner containing version and copyright information.
+
 ---
 
 ## Example Commands
@@ -394,6 +408,10 @@ Clear the default TwitchDownloader cache folder
 
     ./TwitchDownloaderCLI cache --clear
 
+Concatenate several ts files into a single output file
+
+    TwitchDownloaderCLI tsmerge -i list.txt -o output.ts
+
 Print the available operations
 
     ./TwitchDownloaderCLI help
@@ -415,3 +433,6 @@ Default true boolean flags must be assigned: `--default-true-flag=false`. Defaul
 For Linux users, ensure both `fontconfig` and `libfontconfig1` are installed. `apt-get install fontconfig libfontconfig1` on Ubuntu.
 
 Some distros, like Linux Alpine, lack fonts for some languages (Arabic, Persian, Thai, etc.) If this is the case for you, install additional fonts families such as [Noto](https://fonts.google.com/noto/specimen/Noto+Sans) or check your distro's wiki page on fonts as it may have an install command for this specific scenario, such as the [Linux Alpine](https://wiki.alpinelinux.org/wiki/Fonts) font page.
+
+The list file for `tsmerge` may contain relative or absolute paths, with one path per line.
+Alternatively, the list file may also be an M3U8 playlist file.

TwitchDownloaderCore/Options/TsMergeOptions.cs

@@ -0,0 +1,8 @@
+namespace TwitchDownloaderCore.Options
+{
+    public class TsMergeOptions
+    {
+        public string OutputFile { get; set; }
+        public string InputFile { get; set; }
+    }
+}

TwitchDownloaderCore/TsMerger.cs

@@ -0,0 +1,142 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using TwitchDownloaderCore.Options;
+using TwitchDownloaderCore.Tools;
+
+namespace TwitchDownloaderCore
+{
+    public sealed class TsMerger
+    {
+        private readonly TsMergeOptions mergeOptions;
+        private readonly IProgress<ProgressReport> _progress;
+
+        public TsMerger(TsMergeOptions tsMergeOptions, IProgress<ProgressReport> progress)
+        {
+            mergeOptions = tsMergeOptions;
+            _progress = progress;
+        }
+
+        public async Task MergeAsync(CancellationToken cancellationToken)
+        {
+            if (!File.Exists(mergeOptions.InputFile))
+            {
+                throw new FileNotFoundException("Input file does not exist");
+            }
+
+            var isM3U8 = false;
+            var fileList = new List<string>();
+            await using (var fs = File.Open(mergeOptions.InputFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
+            {
+                using var sr = new StreamReader(fs);
+                while (await sr.ReadLineAsync() is { } line)
+                {
+                    if (string.IsNullOrWhiteSpace(line)) continue;
+
+                    if (isM3U8)
+                    {
+                        if (line.StartsWith('#')) continue;
+                    }
+                    else
+                    {
+                        if (line.StartsWith("#EXTM3U")) isM3U8 = true;
+                    }
+
+                    fileList.Add(line);
+                }
+            }
+
+            _progress.Report(new ProgressReport(ReportType.SameLineStatus, "Verifying Parts 0% [1/2]"));
+
+            await VerifyVideoParts(fileList, cancellationToken);
+
+            _progress.Report(new ProgressReport() { ReportType = ReportType.NewLineStatus, Data = "Combining Parts 0% [2/2]" });
+
+            await CombineVideoParts(fileList, cancellationToken);
+
+            _progress.Report(new ProgressReport(100));
+        }
+
+        private async Task VerifyVideoParts(IReadOnlyCollection<string> fileList, CancellationToken cancellationToken)
+        {
+            var failedParts = new List<string>();
+            var partCount = fileList.Count;
+            var doneCount = 0;
+
+            foreach (var part in fileList)
+            {
+                var isValidTs = await VerifyVideoPart(part);
+                if (!isValidTs)
+                {
+                    failedParts.Add(part);
+                }
+
+                doneCount++;
+                var percent = (int)(doneCount / (double)partCount * 100);
+                _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Verifying Parts {percent}% [1/2]"));
+                _progress.Report(new ProgressReport(percent));
+
+                cancellationToken.ThrowIfCancellationRequested();
+            }
+
+            if (failedParts.Count != 0)
+            {
+                if (failedParts.Count == fileList.Count)
+                {
+                    // Every video part returned corrupted, probably a false positive.
+                    return;
+                }
+
+                _progress.Report(new ProgressReport(ReportType.Log, $"The following TS files are invalid or corrupted: {string.Join(", ", failedParts)}"));
+            }
+        }
+
+        private static async Task<bool> VerifyVideoPart(string filePath)
+        {
+            const int TS_PACKET_LENGTH = 188; // MPEG TS packets are made of a header and a body: [ 4B ][   184B   ] - https://tsduck.io/download/docs/mpegts-introduction.pdf
+
+            if (!File.Exists(filePath))
+            {
+                return false;
+            }
+
+            await using var fs = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
+            var fileLength = fs.Length;
+            if (fileLength == 0 || fileLength % TS_PACKET_LENGTH != 0)
+            {
+                return false;
+            }
+
+            return true;
+        }
+
+        private async Task CombineVideoParts(IReadOnlyCollection<string> fileList, CancellationToken cancellationToken)
+        {
+            DriveInfo outputDrive = DriveHelper.GetOutputDrive(mergeOptions.OutputFile);
+            string outputFile = mergeOptions.OutputFile;
+
+            int partCount = fileList.Count;
+            int doneCount = 0;
+
+            await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.Read);
+            foreach (var partFile in fileList)
+            {
+                await DriveHelper.WaitForDrive(outputDrive, _progress, cancellationToken);
+
+                await using (var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.Read))
+                {
+                    await fs.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false);
+                }
+
+                doneCount++;
+                int percent = (int)(doneCount / (double)partCount * 100);
+                _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Combining Parts {percent}% [2/2]"));
+                _progress.Report(new ProgressReport(percent));
+
+                cancellationToken.ThrowIfCancellationRequested();
+            }
+        }
+    }
+}