Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial support for AV1/H.265 VODs #1251

Merged
merged 17 commits into from
Dec 26, 2024
Merged

Initial support for AV1/H.265 VODs #1251

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
73157cf
Support changing stream IDs depending on container type
ScrubN Nov 11, 2024
fdd3611
Fix ByteRange
ScrubN Nov 11, 2024
8a430b8
Support parsing EXT-X-MAP keys
ScrubN Dec 13, 2024
7c492a9
Update M3U8 tests
ScrubN Dec 13, 2024
500299c
Support adding header content to downloaded files
ScrubN Dec 13, 2024
07ad013
Fix potential part count issues
ScrubN Dec 13, 2024
451de5c
Add verbose log when downloading header file
ScrubN Dec 13, 2024
537ecbe
Rename VerifyTsLength to CheckTsLength
ScrubN Dec 13, 2024
8d24058
Fix test variable names
ScrubN Dec 13, 2024
e56a502
Add warning when downloading AV1 VODs
ScrubN Dec 26, 2024
d5e21d1
Merge ByteRange structs
ScrubN Dec 26, 2024
e9d3cb1
Update tests
ScrubN Dec 26, 2024
55c6b66
Fix some M3U8 stringification issues
ScrubN Dec 26, 2024
4422052
Fix log message
ScrubN Dec 26, 2024
c29f5c8
Use IReadOnlyCollection
ScrubN Dec 26, 2024
00c10f1
Use switch statement
ScrubN Dec 26, 2024
67ea209
Do not read header file contents into memory
ScrubN Dec 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 88 additions & 11 deletions TwitchDownloaderCore.Tests/ToolTests/M3U8Tests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,81 @@ public void CorrectlyParsesTwitchM3U8OfTransportStreams(bool useStream, string c
}
}

[Theory]
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
[InlineData(false, "ru-RU")]
[InlineData(true, "ru-RU")]
public void CorrectlyParsesTwitchM3U8OfMp4s(bool useStream, string culture)
{
const string EXAMPLE_M3U8_TWITCH =
"#EXTM3U" +
"\n#EXT-X-VERSION:6" +
"\n#EXT-X-TARGETDURATION:10" +
"\n#ID3-EQUIV-TDTG:2024-12-08T00:12:24" +
"\n#EXT-X-PLAYLIST-TYPE:EVENT" +
"\n#EXT-X-MEDIA-SEQUENCE:0" +
"\n#EXT-X-TWITCH-ELAPSED-SECS:0.000" +
"\n#EXT-X-TWITCH-TOTAL-SECS:1137.134" +
"\n#EXT-X-MAP:URI=\"init-0.mp4\"" +
"\n#EXTINF:10.000,\n0.mp4\n#EXTINF:10.000,\n1.mp4\n#EXTINF:10.000,\n2.mp4\n#EXTINF:10.000,\n3.mp4\n#EXTINF:10.000,\n4.mp4\n#EXTINF:10.000,\n5.mp4\n#EXTINF:10.000,\n6.mp4\n#EXTINF:10.000,\n7.mp4" +
"\n#EXTINF:10.000,\n8.mp4\n#EXTINF:10.000,\n9.mp4\n#EXTINF:10.000,\n10.mp4\n#EXTINF:10.000,\n11.mp4\n#EXTINF:10.000,\n12.mp4\n#EXTINF:10.000,\n13.mp4\n#EXTINF:10.000,\n14.mp4\n#EXTINF:10.000," +
"\n15.mp4\n#EXTINF:10.000,\n16.mp4\n#EXTINF:10.000,\n17.mp4\n#EXTINF:10.000,\n18.mp4\n#EXTINF:10.000,\n19.mp4\n#EXTINF:10.000,\n20.mp4\n#EXTINF:10.000,\n21.mp4\n#EXTINF:10.000,\n22.mp4" +
"\n#EXTINF:10.000,\n23.mp4\n#EXTINF:10.000,\n24.mp4\n#EXTINF:10.000,\n25.mp4\n#EXTINF:10.000,\n26.mp4\n#EXTINF:10.000,\n27.mp4\n#EXTINF:10.000,\n28.mp4\n#EXTINF:10.000,\n29.mp4\n#EXTINF:10.000," +
"\n30.mp4\n#EXTINF:10.000,\n31.mp4\n#EXTINF:10.000,\n32.mp4\n#EXTINF:10.000,\n33.mp4\n#EXTINF:10.000,\n34.mp4\n#EXTINF:10.000,\n35.mp4\n#EXTINF:10.000,\n36.mp4\n#EXTINF:10.000,\n37.mp4" +
"\n#EXTINF:10.000,\n38.mp4\n#EXTINF:10.000,\n39.mp4\n#EXTINF:10.000,\n40.mp4\n#EXTINF:10.000,\n41.mp4\n#EXTINF:10.000,\n42.mp4\n#EXTINF:10.000,\n43.mp4\n#EXTINF:10.000,\n44.mp4\n#EXTINF:10.000," +
"\n45.mp4\n#EXTINF:10.000,\n46.mp4\n#EXTINF:10.000,\n47.mp4\n#EXTINF:10.000,\n48.mp4\n#EXTINF:10.000,\n49.mp4\n#EXTINF:10.000,\n50.mp4\n#EXTINF:10.000,\n51.mp4\n#EXTINF:10.000,\n52.mp4" +
"\n#EXTINF:10.000,\n53.mp4\n#EXTINF:10.000,\n54.mp4\n#EXTINF:10.000,\n55.mp4\n#EXTINF:10.000,\n56.mp4\n#EXTINF:10.000,\n57.mp4\n#EXTINF:10.000,\n58.mp4\n#EXTINF:10.000,\n59.mp4\n#EXTINF:10.000," +
"\n60.mp4\n#EXTINF:10.000,\n61.mp4\n#EXTINF:10.000,\n62.mp4\n#EXTINF:10.000,\n63.mp4\n#EXTINF:10.000,\n64.mp4\n#EXTINF:10.000,\n65.mp4\n#EXTINF:10.000,\n66.mp4\n#EXTINF:10.000,\n67.mp4" +
"\n#EXTINF:10.000,\n68.mp4\n#EXTINF:10.000,\n69.mp4\n#EXTINF:10.000,\n70.mp4\n#EXTINF:10.000,\n71.mp4\n#EXTINF:10.000,\n72.mp4\n#EXTINF:10.000,\n73.mp4\n#EXTINF:10.000,\n74.mp4\n#EXTINF:10.000," +
"\n75.mp4\n#EXTINF:10.000,\n76.mp4\n#EXTINF:10.000,\n77.mp4\n#EXTINF:10.000,\n78.mp4\n#EXTINF:10.000,\n79.mp4\n#EXTINF:10.000,\n80.mp4\n#EXTINF:10.000,\n81.mp4\n#EXTINF:10.000,\n82.mp4" +
"\n#EXTINF:10.000,\n83.mp4\n#EXTINF:10.000,\n84.mp4\n#EXTINF:10.000,\n85.mp4\n#EXTINF:10.000,\n86.mp4\n#EXTINF:10.000,\n87.mp4\n#EXTINF:10.000,\n88.mp4\n#EXTINF:10.000,\n89.mp4\n#EXTINF:10.000," +
"\n90.mp4\n#EXTINF:10.000,\n91.mp4\n#EXTINF:10.000,\n92.mp4\n#EXTINF:10.000,\n93.mp4\n#EXTINF:10.000,\n94.mp4\n#EXTINF:10.000,\n95.mp4\n#EXTINF:10.000,\n96.mp4\n#EXTINF:10.000,\n97.mp4" +
"\n#EXTINF:10.000,\n98.mp4\n#EXTINF:10.000,\n99.mp4\n#EXTINF:10.000,\n100.mp4\n#EXTINF:10.000,\n101.mp4\n#EXTINF:10.000,\n102.mp4\n#EXTINF:10.000,\n103.mp4\n#EXTINF:10.000,\n104.mp4" +
"\n#EXTINF:10.000,\n105.mp4\n#EXTINF:10.000,\n106.mp4\n#EXTINF:10.000,\n107.mp4\n#EXTINF:10.000,\n108.mp4\n#EXTINF:10.000,\n109.mp4\n#EXTINF:10.000,\n110.mp4\n#EXTINF:10.000,\n111.mp4" +
"\n#EXTINF:10.000,\n112.mp4\n#EXTINF:7.134,\n113.mp4\n#EXT-X-ENDLIST";

var oldCulture = CultureInfo.CurrentCulture;
CultureInfo.CurrentCulture = new CultureInfo(culture);

M3U8 m3u8;
if (useStream)
{
var bytes = Encoding.Unicode.GetBytes(EXAMPLE_M3U8_TWITCH);
using var ms = new MemoryStream(bytes);
m3u8 = M3U8.Parse(ms, Encoding.Unicode);
}
else
{
m3u8 = M3U8.Parse(EXAMPLE_M3U8_TWITCH);
}

CultureInfo.CurrentCulture = oldCulture;

Assert.Equal(6u, m3u8.FileMetadata.Version);
Assert.Equal(10u, m3u8.FileMetadata.StreamTargetDuration);
Assert.Equal("2024-12-08T00:12:24", m3u8.FileMetadata.UnparsedValues.FirstOrDefault(x => x.Key == "#ID3-EQUIV-TDTG:").Value);
Assert.Equal(M3U8.Metadata.PlaylistType.Event, m3u8.FileMetadata.Type);
Assert.Equal(0u, m3u8.FileMetadata.MediaSequence);
Assert.Equal("init-0.mp4", m3u8.FileMetadata.Map.Uri);
Assert.Equal(default, m3u8.FileMetadata.Map.ByteRange);
Assert.Equal(0m, m3u8.FileMetadata.TwitchElapsedSeconds);
Assert.Equal(1137.134m, m3u8.FileMetadata.TwitchTotalSeconds);

Assert.Equal(114, m3u8.Streams.Length);

var duration = 1137.134m;
for (var i = 0; i < m3u8.Streams.Length; i++)
{
var stream = m3u8.Streams[i];
Assert.Equal(duration > 10 ? 10 : duration, stream.PartInfo.Duration);
Assert.False(stream.PartInfo.Live);
Assert.Equal($"{i}.mp4", stream.Path);

duration -= 10;
}
}

[Theory]
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
Expand Down Expand Up @@ -301,7 +376,7 @@ public void CorrectlyParsesKickM3U8OfTransportStreams(bool useStream, string cul
"\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:35:03.97Z\n#EXT-X-BYTERANGE:1768140@3175696\n#EXTINF:2.000,\n506.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:35:05.97Z\n#EXT-X-BYTERANGE:1519040@4943836\n#EXTINF:2.000,\n506.ts" +
"\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:35:07.97Z\n#EXT-X-BYTERANGE:1506068@6462876\n#EXTINF:2.000,\n506.ts\n#EXT-X-ENDLIST";

var streamValues = new (DateTimeOffset programDateTime, M3U8.Stream.ExtByteRange byteRange, string path)[]
var streamValues = new (DateTimeOffset programDateTime, M3U8.ByteRange byteRange, string path)[]
{
(DateTimeOffset.Parse("2023-11-16T05:34:07.97Z"), (1601196, 6470396), "500.ts"),
(DateTimeOffset.Parse("2023-11-16T05:34:09.97Z"), (1588224, 0), "501.ts"),
Expand Down Expand Up @@ -474,13 +549,14 @@ public void CorrectlyParsesKickM3U8StreamInfo(string streamInfoString, int bandw
}

[Theory]
[InlineData(100, 200, "100@200")]
[InlineData(100, 200, "#EXT-X-BYTERANGE:100@200")]
public void CorrectlyParsesByteRange(uint start, uint length, string byteRangeString)
[InlineData(100, 200, "100@200", "")]
[InlineData(100, 200, "#EXT-X-BYTERANGE:100@200", "#EXT-X-BYTERANGE:")]
[InlineData(100, 200, "BYTERANGE=100@200", "BYTERANGE=")]
public void CorrectlyParsesByteRange(uint length, uint start, string byteRangeString, string key)
{
var expected = new M3U8.Stream.ExtByteRange(start, length);
var expected = new M3U8.ByteRange(length, start);

var actual = M3U8.Stream.ExtByteRange.Parse(byteRangeString);
var actual = M3U8.ByteRange.Parse(byteRangeString, key);

Assert.Equal(expected, actual);
}
Expand All @@ -491,15 +567,15 @@ public void CorrectlyParsesByteRange(uint start, uint length, string byteRangeSt
[InlineData("42949672950000")]
public void ThrowsFormatExceptionForBadByteRangeString(string byteRangeString)
{
Assert.Throws<FormatException>(() => M3U8.Stream.ExtByteRange.Parse(byteRangeString));
Assert.Throws<FormatException>(() => M3U8.ByteRange.Parse(byteRangeString, default));
}

[Theory]
[InlineData(100, 200, "100x200")]
[InlineData(100, 200, "RESOLUTION=100x200")]
public void CorrectlyParsesResolution(uint start, uint length, string byteRangeString)
public void CorrectlyParsesResolution(uint width, uint height, string byteRangeString)
{
var expected = new M3U8.Stream.ExtStreamInfo.StreamResolution(start, length);
var expected = new M3U8.Stream.ExtStreamInfo.StreamResolution(width, height);

var actual = M3U8.Stream.ExtStreamInfo.StreamResolution.Parse(byteRangeString);

Expand All @@ -510,9 +586,9 @@ public void CorrectlyParsesResolution(uint start, uint length, string byteRangeS
[InlineData("429496729500x1")]
[InlineData("1x429496729500")]
[InlineData("42949672950000")]
public void ThrowsFormatExceptionForBadResolutionString(string byteRangeString)
public void ThrowsFormatExceptionForBadResolutionString(string resolutionString)
{
Assert.Throws<FormatException>(() => M3U8.Stream.ExtStreamInfo.StreamResolution.Parse(byteRangeString));
Assert.Throws<FormatException>(() => M3U8.Stream.ExtStreamInfo.StreamResolution.Parse(resolutionString));
}

[Theory]
Expand Down Expand Up @@ -545,6 +621,7 @@ public void CorrectlyStringifiesInvariantOfCulture(string culture)
"\n#EXT-X-VERSION:4" +
"\n#EXT-X-MEDIA-SEQUENCE:0" +
"\n#EXT-X-TARGETDURATION:2" +
"\n#EXT-X-MAP:URI=\"init-0.mp4\"" +
"\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:07.97Z\n#EXT-X-BYTERANGE:1601196@6470396\n#EXTINF:2.000,\n500.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:09.97Z\n#EXT-X-BYTERANGE:1588224@0\n#EXTINF:2.000,\n501.ts" +
"\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:11.97Z\n#EXT-X-BYTERANGE:1579200@1588224\n#EXTINF:2.000,\n501.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:13.97Z\n#EXT-X-BYTERANGE:1646128@3167424\n#EXTINF:2.000,\n501.ts" +
"\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:15.97Z\n#EXT-X-BYTERANGE:1587472@4813552\n#EXTINF:2.000,\n501.ts\n#EXT-X-PROGRAM-DATE-TIME:2023-11-16T05:34:17.97Z\n#EXT-X-BYTERANGE:1594052@6401024\n#EXTINF:2.000,\n501.ts" +
Expand Down
19 changes: 15 additions & 4 deletions TwitchDownloaderCore/Tools/DownloadTools.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Net.Http;
using System.Threading;
Expand All @@ -15,20 +16,30 @@ public static class DownloadTools
/// <param name="httpClient">The <see cref="HttpClient"/> to perform the download operation.</param>
/// <param name="url">The url of the file to download.</param>
/// <param name="destinationFile">The path to the file where download will be saved.</param>
/// <param name="headerFile">Path to a file whose contents will be written to the start of the destination file.</param>
/// <param name="throttleKib">The maximum download speed in kibibytes per second, or -1 for no maximum.</param>
/// <param name="logger">Logger.</param>
/// <param name="cancellationTokenSource">A <see cref="CancellationTokenSource"/> containing a <see cref="CancellationToken"/> to cancel the operation.</param>
/// <returns>The expected length of the downloaded file, or -1 if the content length header is not present.</returns>
/// <remarks>The <paramref name="cancellationTokenSource"/> may be canceled by this method.</remarks>
public static async Task<long> DownloadFileAsync(HttpClient httpClient, Uri url, string destinationFile, int throttleKib, ITaskLogger logger, CancellationTokenSource cancellationTokenSource = null)
public static async Task<long> DownloadFileAsync(HttpClient httpClient, Uri url, string destinationFile, [AllowNull] string headerFile, int throttleKib, ITaskLogger logger, CancellationTokenSource cancellationTokenSource = null)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
using var request = new HttpRequestMessage(HttpMethod.Get, url);

var cancellationToken = cancellationTokenSource?.Token ?? CancellationToken.None;

using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();

var fileMode = FileMode.Create;
if (!string.IsNullOrWhiteSpace(headerFile))
{
await using var headerFs = new FileStream(headerFile, FileMode.Open, FileAccess.Read, FileShare.Read);
await using var destinationFs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
await headerFs.CopyToAsync(destinationFs, cancellationToken);
fileMode = FileMode.Append;
}

// Why are we setting a CTS CancelAfter timer? See lay295#265
const int SIXTY_SECONDS = 60;
if (throttleKib == -1 || !response.Content.Headers.ContentLength.HasValue)
Expand All @@ -48,7 +59,7 @@ public static async Task<long> DownloadFileAsync(HttpClient httpClient, Uri url,
{
case -1:
{
await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
await using var fs = new FileStream(destinationFile, fileMode, FileAccess.Write, FileShare.Read);
await response.Content.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
break;
}
Expand All @@ -58,7 +69,7 @@ public static async Task<long> DownloadFileAsync(HttpClient httpClient, Uri url,
{
await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken);
await using var throttledStream = new ThrottledStream(contentStream, throttleKib);
await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
await using var fs = new FileStream(destinationFile, fileMode, FileAccess.Write, FileShare.Read);
await throttledStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
}
catch (IOException ex) when (ex.Message.Contains("EOF"))
Expand Down
28 changes: 21 additions & 7 deletions TwitchDownloaderCore/Tools/FfmpegConcatList.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
Expand All @@ -12,7 +13,7 @@ public static class FfmpegConcatList
{
private const string LINE_FEED = "\u000A";

public static async Task SerializeAsync(string filePath, M3U8 playlist, Range videoListCrop, CancellationToken cancellationToken = default)
public static async Task SerializeAsync(string filePath, M3U8 playlist, Range videoListCrop, StreamIds streamIds, CancellationToken cancellationToken = default)
{
await using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read);
await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED };
Expand All @@ -27,16 +28,29 @@ public static async Task SerializeAsync(string filePath, M3U8 playlist, Range vi
await sw.WriteAsync(DownloadTools.RemoveQueryString(stream.Path));
await sw.WriteLineAsync('\'');

await sw.WriteLineAsync("stream");
await sw.WriteLineAsync("exact_stream_id 0x100"); // Audio
await sw.WriteLineAsync("stream");
await sw.WriteLineAsync("exact_stream_id 0x101"); // Video
await sw.WriteLineAsync("stream");
await sw.WriteLineAsync("exact_stream_id 0x102"); // Subtitle
foreach (var id in streamIds.Ids)
{
await sw.WriteLineAsync("stream");
await sw.WriteLineAsync($"exact_stream_id {id}");
}

await sw.WriteAsync("duration ");
await sw.WriteLineAsync(stream.PartInfo.Duration.ToString(CultureInfo.InvariantCulture));
}
}

public record StreamIds
{
public static readonly StreamIds TransportStream = new("0x100", "0x101", "0x102");
public static readonly StreamIds Mp4 = new("0x1", "0x2");
public static readonly StreamIds None = new();

private StreamIds(params string[] ids)
{
Ids = ids;
}

public IEnumerable<string> Ids { get; }
}
}
}
Loading