Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix M3U8 failing to parse when the application is using certain languages #937

Merged
merged 3 commits into from
Jan 6, 2024
Merged

Fix M3U8 failing to parse when the application is using certain languages #937

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7784393
Make number parsing culture invariant
ScrubN Jan 6, 2024
a6dd716
Add alternate culture for M3U8.Parse tests
ScrubN Jan 6, 2024
6cc8232
Add non-throwing result for non-crucial values
ScrubN Jan 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 52 additions & 16 deletions TwitchDownloaderCore.Tests/M3U8Tests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Text;
using System.Globalization;
using System.Text;
using TwitchDownloaderCore.Tools;

namespace TwitchDownloaderCore.Tests
Expand All @@ -7,9 +8,11 @@ namespace TwitchDownloaderCore.Tests
public class M3U8Tests
{
[Theory]
[InlineData(false)]
[InlineData(true)]
public void CorrectlyParsesTwitchM3U8OfTransportStreams(bool useStream)
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
[InlineData(false, "ru-RU")]
[InlineData(true, "ru-RU")]
public void CorrectlyParsesTwitchM3U8OfTransportStreams(bool useStream, string culture)
{
const string ExampleM3U8Twitch =
"#EXTM3U" +
Expand All @@ -28,6 +31,9 @@ public void CorrectlyParsesTwitchM3U8OfTransportStreams(bool useStream)
"\n#EXTINF:10.000,\n40.ts\n#EXTINF:10.000,\n41.ts\n#EXTINF:10.000,\n42.ts\n#EXTINF:10.000,\n43.ts\n#EXTINF:10.000,\n44.ts\n#EXTINF:10.000,\n45.ts\n#EXTINF:10.000,\n46.ts\n#EXTINF:10.000,\n47.ts" +
"\n#EXTINF:10.000,\n48.ts\n#EXTINF:10.000,\n49.ts\n#EXT-X-ENDLIST";

var oldCulture = CultureInfo.CurrentCulture;
CultureInfo.CurrentCulture = new CultureInfo(culture);

M3U8 m3u8;
if (useStream)
{
Expand All @@ -40,6 +46,8 @@ public void CorrectlyParsesTwitchM3U8OfTransportStreams(bool useStream)
m3u8 = M3U8.Parse(ExampleM3U8Twitch);
}

CultureInfo.CurrentCulture = oldCulture;

Assert.Equal(3u, m3u8.FileMetadata.Version);
Assert.Equal(10u, m3u8.FileMetadata.StreamTargetDuration);
Assert.Equal("2023-09-23T17:37:06", m3u8.FileMetadata.UnparsedValues.FirstOrDefault(x => x.Key == "#ID3-EQUIV-TDTG:").Value);
Expand All @@ -59,9 +67,11 @@ public void CorrectlyParsesTwitchM3U8OfTransportStreams(bool useStream)
}

[Theory]
[InlineData(false)]
[InlineData(true)]
public void CorrectlyParsesTwitchM3U8OfLiveStreams(bool useStream)
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
[InlineData(false, "ru-RU")]
[InlineData(true, "ru-RU")]
public void CorrectlyParsesTwitchM3U8OfLiveStreams(bool useStream, string culture)
{
const string ExampleM3U8Twitch =
"#EXTM3U" +
Expand Down Expand Up @@ -113,6 +123,9 @@ public void CorrectlyParsesTwitchM3U8OfLiveStreams(bool useStream)
(DateTimeOffset.Parse("2023-09-17T02:32:16.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/hij-567KLM_890.ts")
};

var oldCulture = CultureInfo.CurrentCulture;
CultureInfo.CurrentCulture = new CultureInfo(culture);

M3U8 m3u8;
if (useStream)
{
Expand All @@ -125,6 +138,8 @@ public void CorrectlyParsesTwitchM3U8OfLiveStreams(bool useStream)
m3u8 = M3U8.Parse(ExampleM3U8Twitch);
}

CultureInfo.CurrentCulture = oldCulture;

Assert.Equal(3u, m3u8.FileMetadata.Version);
Assert.Equal(5u, m3u8.FileMetadata.StreamTargetDuration);
Assert.Equal(M3U8.Metadata.PlaylistType.Unknown, m3u8.FileMetadata.Type);
Expand All @@ -146,9 +161,11 @@ public void CorrectlyParsesTwitchM3U8OfLiveStreams(bool useStream)
}

[Theory]
[InlineData(false)]
[InlineData(true)]
public void CorrectlyParsesTwitchM3U8OfPlaylists(bool useStream)
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
[InlineData(false, "ru-RU")]
[InlineData(true, "ru-RU")]
public void CorrectlyParsesTwitchM3U8OfPlaylists(bool useStream, string culture)
{
const string ExampleM3U8Twitch =
"#EXTM3U" +
Expand Down Expand Up @@ -194,6 +211,9 @@ public void CorrectlyParsesTwitchM3U8OfPlaylists(bool useStream)
"https://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/160p30/index-dvr.m3u8")
};

var oldCulture = CultureInfo.CurrentCulture;
CultureInfo.CurrentCulture = new CultureInfo(culture);

M3U8 m3u8;
if (useStream)
{
Expand All @@ -206,6 +226,8 @@ public void CorrectlyParsesTwitchM3U8OfPlaylists(bool useStream)
m3u8 = M3U8.Parse(ExampleM3U8Twitch);
}

CultureInfo.CurrentCulture = oldCulture;

Assert.Equal(streams.Length, m3u8.Streams.Length);
Assert.Equivalent(streams[0], m3u8.Streams[0], true);
Assert.Equivalent(streams[1], m3u8.Streams[1], true);
Expand Down Expand Up @@ -252,9 +274,11 @@ public void CorrectlyParsesTwitchM3U8StreamInfo(string streamInfoString, int ban
}

[Theory]
[InlineData(false)]
[InlineData(true)]
public void CorrectlyParsesKickM3U8OfTransportStreams(bool useStream)
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
[InlineData(false, "ru-RU")]
[InlineData(true, "ru-RU")]
public void CorrectlyParsesKickM3U8OfTransportStreams(bool useStream, string culture)
{
const string ExampleM3U8Kick =
"#EXTM3U" +
Expand Down Expand Up @@ -313,6 +337,9 @@ public void CorrectlyParsesKickM3U8OfTransportStreams(bool useStream)
(DateTimeOffset.Parse("2023-11-16T05:35:07.97Z"), (1506068, 6462876), "506.ts")
};

var oldCulture = CultureInfo.CurrentCulture;
CultureInfo.CurrentCulture = new CultureInfo(culture);

M3U8 m3u8;
if (useStream)
{
Expand All @@ -325,6 +352,8 @@ public void CorrectlyParsesKickM3U8OfTransportStreams(bool useStream)
m3u8 = M3U8.Parse(ExampleM3U8Kick);
}

CultureInfo.CurrentCulture = oldCulture;

Assert.Equal(4u, m3u8.FileMetadata.Version);
Assert.Equal(2u, m3u8.FileMetadata.StreamTargetDuration);
Assert.Equal(M3U8.Metadata.PlaylistType.Unknown, m3u8.FileMetadata.Type);
Expand All @@ -343,9 +372,11 @@ public void CorrectlyParsesKickM3U8OfTransportStreams(bool useStream)
}

[Theory]
[InlineData(false)]
[InlineData(true)]
public void CorrectlyParsesKickM3U8OfPlaylists(bool useStream)
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
[InlineData(false, "ru-RU")]
[InlineData(true, "ru-RU")]
public void CorrectlyParsesKickM3U8OfPlaylists(bool useStream, string culture)
{
const string ExampleM3U8Kick =
"#EXTM3U" +
Expand Down Expand Up @@ -386,6 +417,9 @@ public void CorrectlyParsesKickM3U8OfPlaylists(bool useStream)
"160p30/playlist.m3u8")
};

var oldCulture = CultureInfo.CurrentCulture;
CultureInfo.CurrentCulture = new CultureInfo(culture);

M3U8 m3u8;
if (useStream)
{
Expand All @@ -398,6 +432,8 @@ public void CorrectlyParsesKickM3U8OfPlaylists(bool useStream)
m3u8 = M3U8.Parse(ExampleM3U8Kick);
}

CultureInfo.CurrentCulture = oldCulture;

Assert.Equal(streams.Length, m3u8.Streams.Length);
Assert.Equivalent(streams[0], m3u8.Streams[0], true);
Assert.Equivalent(streams[1], m3u8.Streams[1], true);
Expand Down
49 changes: 33 additions & 16 deletions TwitchDownloaderCore/Tools/M3U8.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
Expand Down Expand Up @@ -149,7 +151,7 @@ private static bool ParseM3U8Key(ReadOnlySpan<char> text, Metadata.Builder metad
}
else if (text.StartsWith(PROGRAM_DATE_TIME_KEY))
{
extProgramDateTime = ParsingHelpers.ParseDateTimeOffset(text, PROGRAM_DATE_TIME_KEY);
extProgramDateTime = ParsingHelpers.ParseDateTimeOffset(text, PROGRAM_DATE_TIME_KEY, false);
}
else if (text.StartsWith(Stream.ExtByteRange.BYTE_RANGE_KEY))
{
Expand Down Expand Up @@ -339,10 +341,10 @@ public static ExtByteRange Parse(ReadOnlySpan<char> text)
if (separatorIndex == -1)
throw new FormatException($"Unable to parse ByteRange from {text}.");

if (!uint.TryParse(text[..separatorIndex], out var start))
if (!uint.TryParse(text[..separatorIndex], NumberStyles.Integer, CultureInfo.InvariantCulture, out var start))
throw new FormatException($"Unable to parse ByteRange from {text}.");

if (!uint.TryParse(text[(separatorIndex + 1)..], out var end))
if (!uint.TryParse(text[(separatorIndex + 1)..], NumberStyles.Integer, CultureInfo.InvariantCulture, out var end))
throw new FormatException($"Unable to parse ByteRange from {text}.");

return new ExtByteRange(start, end);
Expand Down Expand Up @@ -488,10 +490,10 @@ public static StreamResolution Parse(ReadOnlySpan<char> text)
if (separatorIndex == -1 || separatorIndex == text.Length)
throw new FormatException($"Unable to parse Resolution from {text}.");

if (!uint.TryParse(text[..separatorIndex], out var width))
if (!uint.TryParse(text[..separatorIndex], NumberStyles.Integer, CultureInfo.InvariantCulture, out var width))
throw new FormatException($"Unable to parse Resolution from {text}.");

if (!uint.TryParse(text[(separatorIndex + 1)..], out var height))
if (!uint.TryParse(text[(separatorIndex + 1)..], NumberStyles.Integer, CultureInfo.InvariantCulture, out var height))
throw new FormatException($"Unable to parse Resolution from {text}.");

return new StreamResolution(width, height);
Expand Down Expand Up @@ -588,11 +590,11 @@ public static ExtStreamInfo Parse(ReadOnlySpan<char> text)

if (text.StartsWith(KEY_PROGRAM_ID))
{
streamInfo.ProgramId = ParsingHelpers.ParseIntValue(text, KEY_PROGRAM_ID);
streamInfo.ProgramId = ParsingHelpers.ParseIntValue(text, KEY_PROGRAM_ID, false);
}
else if (text.StartsWith(KEY_BANDWIDTH))
{
streamInfo.Bandwidth = ParsingHelpers.ParseIntValue(text, KEY_BANDWIDTH);
streamInfo.Bandwidth = ParsingHelpers.ParseIntValue(text, KEY_BANDWIDTH, false);
}
else if (text.StartsWith(KEY_CODECS))
{
Expand All @@ -608,7 +610,7 @@ public static ExtStreamInfo Parse(ReadOnlySpan<char> text)
}
else if (text.StartsWith(KEY_FRAMERATE))
{
streamInfo.Framerate = ParsingHelpers.ParseDecimalValue(text, KEY_FRAMERATE);
streamInfo.Framerate = ParsingHelpers.ParseDecimalValue(text, KEY_FRAMERATE, false);
}

var nextIndex = text.UnEscapedIndexOf(',');
Expand Down Expand Up @@ -711,40 +713,49 @@ public static string ParseStringValue(ReadOnlySpan<char> text, ReadOnlySpan<char
return temp[..closeQuote].ToString();
}

public static int ParseIntValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName)
public static int ParseIntValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName, bool strict = true)
{
var temp = text[keyName.Length..];
temp = temp[..NextKeyStart(temp)];

if (int.TryParse(temp, out var intValue))
if (int.TryParse(temp, NumberStyles.Integer, CultureInfo.InvariantCulture, out var intValue))
return intValue;

if (!strict)
return default;

throw new FormatException($"Unable to parse integer from: {text}");
}

public static uint ParseUIntValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName)
public static uint ParseUIntValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName, bool strict = true)
{
var temp = text[keyName.Length..];
temp = temp[..NextKeyStart(temp)];

if (uint.TryParse(temp, out var uIntValue))
if (uint.TryParse(temp, NumberStyles.Integer, CultureInfo.InvariantCulture, out var uIntValue))
return uIntValue;

if (!strict)
return default;

throw new FormatException($"Unable to parse integer from: {text}");
}

public static decimal ParseDecimalValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName)
public static decimal ParseDecimalValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName, bool strict = true)
{
var temp = text[keyName.Length..];
temp = temp[..NextKeyStart(temp)];

if (decimal.TryParse(temp, out var decimalValue))
if (decimal.TryParse(temp, NumberStyles.Number, CultureInfo.InvariantCulture, out var decimalValue))
return decimalValue;

if (!strict)
return default;

throw new FormatException($"Unable to parse decimal from: {text}");
}

public static bool ParseBooleanValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName)
public static bool ParseBooleanValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName, bool strict = true)
{
var temp = text[keyName.Length..];

Expand All @@ -759,6 +770,9 @@ public static bool ParseBooleanValue(ReadOnlySpan<char> text, ReadOnlySpan<char>
if (bool.TryParse(temp, out var booleanValue))
return booleanValue;

if (!strict)
return default;

throw new FormatException($"Unable to parse boolean from: {text}");
}

Expand All @@ -770,14 +784,17 @@ public static Stream.ExtStreamInfo.StreamResolution ParseResolution(ReadOnlySpan
return Stream.ExtStreamInfo.StreamResolution.Parse(temp);
}

public static DateTimeOffset ParseDateTimeOffset(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName)
public static DateTimeOffset ParseDateTimeOffset(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName, bool strict = true)
{
var temp = text[keyName.Length..];
temp = temp[..NextKeyStart(temp)];

if (DateTimeOffset.TryParse(temp, out var dateTimeOffset))
return dateTimeOffset;

if (!strict)
return default;

throw new FormatException($"Unable to parse DateTimeOffset from: {text}");
}

Expand Down
Loading