Skip to content

Commit

Permalink
Fix M3U8 failing to parse when the application is using certain langu…
Browse files Browse the repository at this point in the history 
…ages (#937)

* Make number parsing culture invariant

* Add alternate culture for M3U8.Parse tests

* Add non-throwing result for non-crucial values
  • Loading branch information
@ScrubN
ScrubN authored Jan 6, 2024
1 parent e48a04b commit f166581
Show file tree
Hide file tree
Showing 2 changed files with 85 additions and 32 deletions.
68 changes: 52 additions & 16 deletions TwitchDownloaderCore.Tests/M3U8Tests.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Text;
using System.Globalization;
using System.Text;
using TwitchDownloaderCore.Tools;

namespace TwitchDownloaderCore.Tests
Expand All @@ -7,9 +8,11 @@ namespace TwitchDownloaderCore.Tests
public class M3U8Tests
{
[Theory]
[InlineData(false)]
[InlineData(true)]
public void CorrectlyParsesTwitchM3U8OfTransportStreams(bool useStream)
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
[InlineData(false, "ru-RU")]
[InlineData(true, "ru-RU")]
public void CorrectlyParsesTwitchM3U8OfTransportStreams(bool useStream, string culture)
{
const string ExampleM3U8Twitch =
"#EXTM3U" +
Expand All @@ -28,6 +31,9 @@ public void CorrectlyParsesTwitchM3U8OfTransportStreams(bool useStream)
"\n#EXTINF:10.000,\n40.ts\n#EXTINF:10.000,\n41.ts\n#EXTINF:10.000,\n42.ts\n#EXTINF:10.000,\n43.ts\n#EXTINF:10.000,\n44.ts\n#EXTINF:10.000,\n45.ts\n#EXTINF:10.000,\n46.ts\n#EXTINF:10.000,\n47.ts" +
"\n#EXTINF:10.000,\n48.ts\n#EXTINF:10.000,\n49.ts\n#EXT-X-ENDLIST";

var oldCulture = CultureInfo.CurrentCulture;
CultureInfo.CurrentCulture = new CultureInfo(culture);

M3U8 m3u8;
if (useStream)
{
Expand All @@ -40,6 +46,8 @@ public void CorrectlyParsesTwitchM3U8OfTransportStreams(bool useStream)
m3u8 = M3U8.Parse(ExampleM3U8Twitch);
}

CultureInfo.CurrentCulture = oldCulture;

Assert.Equal(3u, m3u8.FileMetadata.Version);
Assert.Equal(10u, m3u8.FileMetadata.StreamTargetDuration);
Assert.Equal("2023-09-23T17:37:06", m3u8.FileMetadata.UnparsedValues.FirstOrDefault(x => x.Key == "#ID3-EQUIV-TDTG:").Value);
Expand All @@ -59,9 +67,11 @@ public void CorrectlyParsesTwitchM3U8OfTransportStreams(bool useStream)
}

[Theory]
[InlineData(false)]
[InlineData(true)]
public void CorrectlyParsesTwitchM3U8OfLiveStreams(bool useStream)
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
[InlineData(false, "ru-RU")]
[InlineData(true, "ru-RU")]
public void CorrectlyParsesTwitchM3U8OfLiveStreams(bool useStream, string culture)
{
const string ExampleM3U8Twitch =
"#EXTM3U" +
Expand Down Expand Up @@ -113,6 +123,9 @@ public void CorrectlyParsesTwitchM3U8OfLiveStreams(bool useStream)
(DateTimeOffset.Parse("2023-09-17T02:32:16.242Z"), 2.000m, true, "https://video-edge-foo.bar.abs.hls.ttvnw.net/v1/segment/hij-567KLM_890.ts")
};

var oldCulture = CultureInfo.CurrentCulture;
CultureInfo.CurrentCulture = new CultureInfo(culture);

M3U8 m3u8;
if (useStream)
{
Expand All @@ -125,6 +138,8 @@ public void CorrectlyParsesTwitchM3U8OfLiveStreams(bool useStream)
m3u8 = M3U8.Parse(ExampleM3U8Twitch);
}

CultureInfo.CurrentCulture = oldCulture;

Assert.Equal(3u, m3u8.FileMetadata.Version);
Assert.Equal(5u, m3u8.FileMetadata.StreamTargetDuration);
Assert.Equal(M3U8.Metadata.PlaylistType.Unknown, m3u8.FileMetadata.Type);
Expand All @@ -146,9 +161,11 @@ public void CorrectlyParsesTwitchM3U8OfLiveStreams(bool useStream)
}

[Theory]
[InlineData(false)]
[InlineData(true)]
public void CorrectlyParsesTwitchM3U8OfPlaylists(bool useStream)
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
[InlineData(false, "ru-RU")]
[InlineData(true, "ru-RU")]
public void CorrectlyParsesTwitchM3U8OfPlaylists(bool useStream, string culture)
{
const string ExampleM3U8Twitch =
"#EXTM3U" +
Expand Down Expand Up @@ -194,6 +211,9 @@ public void CorrectlyParsesTwitchM3U8OfPlaylists(bool useStream)
"https://abc123def456gh.cloudfront.net/123abc456def789ghi01_streamer42_12345678901_1234567890/160p30/index-dvr.m3u8")
};

var oldCulture = CultureInfo.CurrentCulture;
CultureInfo.CurrentCulture = new CultureInfo(culture);

M3U8 m3u8;
if (useStream)
{
Expand All @@ -206,6 +226,8 @@ public void CorrectlyParsesTwitchM3U8OfPlaylists(bool useStream)
m3u8 = M3U8.Parse(ExampleM3U8Twitch);
}

CultureInfo.CurrentCulture = oldCulture;

Assert.Equal(streams.Length, m3u8.Streams.Length);
Assert.Equivalent(streams[0], m3u8.Streams[0], true);
Assert.Equivalent(streams[1], m3u8.Streams[1], true);
Expand Down Expand Up @@ -252,9 +274,11 @@ public void CorrectlyParsesTwitchM3U8StreamInfo(string streamInfoString, int ban
}

[Theory]
[InlineData(false)]
[InlineData(true)]
public void CorrectlyParsesKickM3U8OfTransportStreams(bool useStream)
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
[InlineData(false, "ru-RU")]
[InlineData(true, "ru-RU")]
public void CorrectlyParsesKickM3U8OfTransportStreams(bool useStream, string culture)
{
const string ExampleM3U8Kick =
"#EXTM3U" +
Expand Down Expand Up @@ -313,6 +337,9 @@ public void CorrectlyParsesKickM3U8OfTransportStreams(bool useStream)
(DateTimeOffset.Parse("2023-11-16T05:35:07.97Z"), (1506068, 6462876), "506.ts")
};

var oldCulture = CultureInfo.CurrentCulture;
CultureInfo.CurrentCulture = new CultureInfo(culture);

M3U8 m3u8;
if (useStream)
{
Expand All @@ -325,6 +352,8 @@ public void CorrectlyParsesKickM3U8OfTransportStreams(bool useStream)
m3u8 = M3U8.Parse(ExampleM3U8Kick);
}

CultureInfo.CurrentCulture = oldCulture;

Assert.Equal(4u, m3u8.FileMetadata.Version);
Assert.Equal(2u, m3u8.FileMetadata.StreamTargetDuration);
Assert.Equal(M3U8.Metadata.PlaylistType.Unknown, m3u8.FileMetadata.Type);
Expand All @@ -343,9 +372,11 @@ public void CorrectlyParsesKickM3U8OfTransportStreams(bool useStream)
}

[Theory]
[InlineData(false)]
[InlineData(true)]
public void CorrectlyParsesKickM3U8OfPlaylists(bool useStream)
[InlineData(false, "en-US")]
[InlineData(true, "en-US")]
[InlineData(false, "ru-RU")]
[InlineData(true, "ru-RU")]
public void CorrectlyParsesKickM3U8OfPlaylists(bool useStream, string culture)
{
const string ExampleM3U8Kick =
"#EXTM3U" +
Expand Down Expand Up @@ -386,6 +417,9 @@ public void CorrectlyParsesKickM3U8OfPlaylists(bool useStream)
"160p30/playlist.m3u8")
};

var oldCulture = CultureInfo.CurrentCulture;
CultureInfo.CurrentCulture = new CultureInfo(culture);

M3U8 m3u8;
if (useStream)
{
Expand All @@ -398,6 +432,8 @@ public void CorrectlyParsesKickM3U8OfPlaylists(bool useStream)
m3u8 = M3U8.Parse(ExampleM3U8Kick);
}

CultureInfo.CurrentCulture = oldCulture;

Assert.Equal(streams.Length, m3u8.Streams.Length);
Assert.Equivalent(streams[0], m3u8.Streams[0], true);
Assert.Equivalent(streams[1], m3u8.Streams[1], true);
Expand Down
49 changes: 33 additions & 16 deletions TwitchDownloaderCore/Tools/M3U8.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
Expand Down Expand Up @@ -149,7 +151,7 @@ private static bool ParseM3U8Key(ReadOnlySpan<char> text, Metadata.Builder metad
}
else if (text.StartsWith(PROGRAM_DATE_TIME_KEY))
{
extProgramDateTime = ParsingHelpers.ParseDateTimeOffset(text, PROGRAM_DATE_TIME_KEY);
extProgramDateTime = ParsingHelpers.ParseDateTimeOffset(text, PROGRAM_DATE_TIME_KEY, false);
}
else if (text.StartsWith(Stream.ExtByteRange.BYTE_RANGE_KEY))
{
Expand Down Expand Up @@ -339,10 +341,10 @@ public static ExtByteRange Parse(ReadOnlySpan<char> text)
if (separatorIndex == -1)
throw new FormatException($"Unable to parse ByteRange from {text}.");

if (!uint.TryParse(text[..separatorIndex], out var start))
if (!uint.TryParse(text[..separatorIndex], NumberStyles.Integer, CultureInfo.InvariantCulture, out var start))
throw new FormatException($"Unable to parse ByteRange from {text}.");

if (!uint.TryParse(text[(separatorIndex + 1)..], out var end))
if (!uint.TryParse(text[(separatorIndex + 1)..], NumberStyles.Integer, CultureInfo.InvariantCulture, out var end))
throw new FormatException($"Unable to parse ByteRange from {text}.");

return new ExtByteRange(start, end);
Expand Down Expand Up @@ -488,10 +490,10 @@ public static StreamResolution Parse(ReadOnlySpan<char> text)
if (separatorIndex == -1 || separatorIndex == text.Length)
throw new FormatException($"Unable to parse Resolution from {text}.");

if (!uint.TryParse(text[..separatorIndex], out var width))
if (!uint.TryParse(text[..separatorIndex], NumberStyles.Integer, CultureInfo.InvariantCulture, out var width))
throw new FormatException($"Unable to parse Resolution from {text}.");

if (!uint.TryParse(text[(separatorIndex + 1)..], out var height))
if (!uint.TryParse(text[(separatorIndex + 1)..], NumberStyles.Integer, CultureInfo.InvariantCulture, out var height))
throw new FormatException($"Unable to parse Resolution from {text}.");

return new StreamResolution(width, height);
Expand Down Expand Up @@ -588,11 +590,11 @@ public static ExtStreamInfo Parse(ReadOnlySpan<char> text)

if (text.StartsWith(KEY_PROGRAM_ID))
{
streamInfo.ProgramId = ParsingHelpers.ParseIntValue(text, KEY_PROGRAM_ID);
streamInfo.ProgramId = ParsingHelpers.ParseIntValue(text, KEY_PROGRAM_ID, false);
}
else if (text.StartsWith(KEY_BANDWIDTH))
{
streamInfo.Bandwidth = ParsingHelpers.ParseIntValue(text, KEY_BANDWIDTH);
streamInfo.Bandwidth = ParsingHelpers.ParseIntValue(text, KEY_BANDWIDTH, false);
}
else if (text.StartsWith(KEY_CODECS))
{
Expand All @@ -608,7 +610,7 @@ public static ExtStreamInfo Parse(ReadOnlySpan<char> text)
}
else if (text.StartsWith(KEY_FRAMERATE))
{
streamInfo.Framerate = ParsingHelpers.ParseDecimalValue(text, KEY_FRAMERATE);
streamInfo.Framerate = ParsingHelpers.ParseDecimalValue(text, KEY_FRAMERATE, false);
}

var nextIndex = text.UnEscapedIndexOf(',');
Expand Down Expand Up @@ -711,40 +713,49 @@ public static string ParseStringValue(ReadOnlySpan<char> text, ReadOnlySpan<char
return temp[..closeQuote].ToString();
}

public static int ParseIntValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName)
public static int ParseIntValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName, bool strict = true)
{
var temp = text[keyName.Length..];
temp = temp[..NextKeyStart(temp)];

if (int.TryParse(temp, out var intValue))
if (int.TryParse(temp, NumberStyles.Integer, CultureInfo.InvariantCulture, out var intValue))
return intValue;

if (!strict)
return default;

throw new FormatException($"Unable to parse integer from: {text}");
}

public static uint ParseUIntValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName)
public static uint ParseUIntValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName, bool strict = true)
{
var temp = text[keyName.Length..];
temp = temp[..NextKeyStart(temp)];

if (uint.TryParse(temp, out var uIntValue))
if (uint.TryParse(temp, NumberStyles.Integer, CultureInfo.InvariantCulture, out var uIntValue))
return uIntValue;

if (!strict)
return default;

throw new FormatException($"Unable to parse integer from: {text}");
}

public static decimal ParseDecimalValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName)
public static decimal ParseDecimalValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName, bool strict = true)
{
var temp = text[keyName.Length..];
temp = temp[..NextKeyStart(temp)];

if (decimal.TryParse(temp, out var decimalValue))
if (decimal.TryParse(temp, NumberStyles.Number, CultureInfo.InvariantCulture, out var decimalValue))
return decimalValue;

if (!strict)
return default;

throw new FormatException($"Unable to parse decimal from: {text}");
}

public static bool ParseBooleanValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName)
public static bool ParseBooleanValue(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName, bool strict = true)
{
var temp = text[keyName.Length..];

Expand All @@ -759,6 +770,9 @@ public static bool ParseBooleanValue(ReadOnlySpan<char> text, ReadOnlySpan<char>
if (bool.TryParse(temp, out var booleanValue))
return booleanValue;

if (!strict)
return default;

throw new FormatException($"Unable to parse boolean from: {text}");
}

Expand All @@ -770,14 +784,17 @@ public static Stream.ExtStreamInfo.StreamResolution ParseResolution(ReadOnlySpan
return Stream.ExtStreamInfo.StreamResolution.Parse(temp);
}

public static DateTimeOffset ParseDateTimeOffset(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName)
public static DateTimeOffset ParseDateTimeOffset(ReadOnlySpan<char> text, ReadOnlySpan<char> keyName, bool strict = true)
{
var temp = text[keyName.Length..];
temp = temp[..NextKeyStart(temp)];

if (DateTimeOffset.TryParse(temp, out var dateTimeOffset))
return dateTimeOffset;

if (!strict)
return default;

throw new FormatException($"Unable to parse DateTimeOffset from: {text}");
}

Expand Down

0 comments on commit f166581

Please sign in to comment.