Skip to content

Commit

Permalink
[BACKEND] Properly sanitize recursive variables (#477)
Browse files Browse the repository at this point in the history
* [BACKEND] Properly sanitize recursive variables

* lint

* update tests
  • Loading branch information
jmbannon authored Feb 25, 2023
1 parent a4dc61e commit a2808fa
Show file tree
Hide file tree
Showing 20 changed files with 69 additions and 60 deletions.
59 changes: 28 additions & 31 deletions src/ytdl_sub/validators/string_formatter_validators.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,12 +134,24 @@ def format_string(self) -> str:
"""
return self._value

def __apply_formatter(
def _apply_formatter(
self, formatter: "StringFormatterValidator", variable_dict: Dict[str, str]
) -> "StringFormatterValidator":
# Ensure the variable names exist within the entry and overrides
for variable_name in formatter.format_variables:
if variable_name not in variable_dict:
# If the variable exists, but is sanitized...
if (
variable_name.endswith("_sanitized")
and variable_name.removesuffix("_sanitized") in variable_dict
):
# Resolve just the non-sanitized version, then sanitize it
variable_dict[variable_name] = sanitize_filename(
StringFormatterValidator(
name=self._name, value=f"{{{variable_name.removesuffix('_sanitized')}}}"
).apply_formatter(variable_dict)
)
# If the variable doesn't exist, error
elif variable_name not in variable_dict:
available_fields = ", ".join(sorted(variable_dict.keys()))
raise self._validation_exception(
self._variable_not_found_error_msg_formatter.format(
Expand All @@ -153,25 +165,25 @@ def __apply_formatter(
value=formatter.format_string.format(**OrderedDict(variable_dict)),
)

def _apply_formatter(self, variable_dict: Dict[str, str], resolve_sanitized: bool = False):
def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
"""
Calls `format` on the format string using the variable_dict as input kwargs
Parameters
----------
variable_dict
kwargs to pass to the format string
Returns
-------
Format string formatted
"""
formatter = self
recursion_depth = 0
max_depth = self._max_format_recursion

if resolve_sanitized:
for format_variable in formatter.format_variables:
# Must resolve the sanitized variable completely
if format_variable.endswith("_sanitized"):
# pylint: disable=protected-access
variable_dict[format_variable] = sanitize_filename(
StringFormatterValidator(
name=self._name, value=f"{{{format_variable}}}"
)._apply_formatter(variable_dict, resolve_sanitized=False)
)
# pylint: enable=protected-access

while formatter.format_variables and recursion_depth < max_depth:
formatter = self.__apply_formatter(formatter=formatter, variable_dict=variable_dict)
formatter = self._apply_formatter(formatter=formatter, variable_dict=variable_dict)
recursion_depth += 1

if formatter.format_variables:
Expand All @@ -184,21 +196,6 @@ def _apply_formatter(self, variable_dict: Dict[str, str], resolve_sanitized: boo

return formatter.format_string

def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
"""
Calls `format` on the format string using the variable_dict as input kwargs
Parameters
----------
variable_dict
kwargs to pass to the format string
Returns
-------
Format string formatted
"""
return self._apply_formatter(variable_dict=variable_dict, resolve_sanitized=True)


# pylint: disable=line-too-long
class OverridesStringFormatterValidator(StringFormatterValidator):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
".ytdl-sub-chapters_from_comments-download-archive.json": "122723ce8d257eebb05178daa26141f6",
"JMC/JMC - Move 78 - Automated Improvisation [Full Album]-thumb.jpg": "c12e6a6f242680d1096a1a99d74a62c6",
"JMC/JMC - Move 78 - Automated Improvisation [Full Album].info.json": "6654b2b5bbf4e7dfb51f2c5e3138df63",
"JMC/JMC - Move 78 - Automated Improvisation [Full Album].mp4": "5bd2291db1ee7848ff5d8ed0fdb98782",
"JMC/JMC - Move 78 - Automated Improvisation [Full Album].info.json": "6fece242a390ce0756f39aab731f47f4",
"JMC/JMC - Move 78 - Automated Improvisation [Full Album].mp4": "40a74798dddf486a656c761516eb0528",
"JMC/JMC - Move 78 - Automated Improvisation [Full Album].nfo": "7a65b184d24c68fc0ec5380432250f5b"
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
".ytdl-sub-file_convert_test-download-archive.json": "65aa49c7345f9fc201e42ddc4a96b19b",
"Beyond The Guitar/Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3-thumb.jpg": "662fcaadf6e80d63591bac19a5fdffb0",
"Beyond The Guitar/Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.info.json": "bdc0ecb04fb478ecf787ddbc64325a2b",
"Beyond The Guitar/Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.info.json": "90d8f639520d8295e009c41f328180a0",
"Beyond The Guitar/Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.mp4": "390177bcc076e309d1534b8bb5afdcc7",
"Beyond The Guitar/Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.nfo": "cacf09ab38f9b3085da9c5af516cf22a"
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
".ytdl-sub-file_convert_test-download-archive.json": "74813dccf4e9732e49f5dc3c2d66f3be",
"Beyond The Guitar/Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3-thumb.jpg": "662fcaadf6e80d63591bac19a5fdffb0",
"Beyond The Guitar/Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.info.json": "3022131504a9df5d2fce266fada1ee3b",
"Beyond The Guitar/Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.info.json": "bd0925c1975a669ec298c810719b9c43",
"Beyond The Guitar/Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.mkv": "b5f248b560f89f3f2a83fcdcd197d486",
"Beyond The Guitar/Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.nfo": "cacf09ab38f9b3085da9c5af516cf22a"
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
{
".ytdl-sub-split_by_chapters_video-download-archive.json": "5e7a31ec4a73e71696f8f3ba9eab9303",
".ytdl-sub-split_by_chapters_video-download-archive.json": "c3fb0b4f31caaa10ac7954ea93da33c4",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/01 - 01. Intro (Feat. Racheal Ofori & Barney Artist).mp3": "29d09da874133ce5c5f7459c2fa6cd85",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/02 - 02. Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "d5432e6a4f7af9810b0e7c8eebe4898a",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/03 - 03. Blaze (Feat. Kaya Thomas - Dyke).mp3": "cf03da6688a73373f9295dc595156e11",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/04 - 04. What If (Interlude).mp3": "b7b943b6f3395c05433b8532364d63d6",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/05 - 05. No Peace (Feat. Tom Misch).mp3": "0d9719268900d4abcd8c0f51ad3af92c",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/06 - 06. Closer (Feat. Lester Duval).mp3": "bbd9ec616a0f7d3c5d13c04bb118681e",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/07 - 07. Delusions: Rumination (Interlude) (Feat. Racheal Ofori).mp3": "2e59ecdc0f8050bcf190a7898d7ae968",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/07 - 07. Delusions Rumination (Interlude) (Feat. Racheal Ofori).mp3": "2e59ecdc0f8050bcf190a7898d7ae968",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/08 - 08. Dreams (Feat. Carmody).mp3": "e58966f3326b876f0ab97459b681f76f",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/09 - 09. Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "d6939215b73457fce4e026f8c9b57ecc",
"Proved Records/[2017] Alfa Mist - Nocturne [Full Album]/10 - 10. Hopeful (Feat. Jordan Rakei).mp3": "82d0e7d8070fd15d71c2e11c0285d426",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
".ytdl-sub-split_by_chapters_with_regex_video-download-archive.json": "ed212171889707bf65462878248fc0b8",
"Project Zombie/[2010] Oblivion Mod \"Falcor\" p.1/01 - Oblivion Mod \"Falcor\" p.1.mp3": "a3c01f164eeca4541aeed49264d2fc8c",
"Project Zombie/[2010] Oblivion Mod \"Falcor\" p.1/folder.jpg": "fb95b510681676e81c321171fc23143e"
".ytdl-sub-split_by_chapters_with_regex_video-download-archive.json": "4008e43668447f1a3a6a55520a6ff475",
"Project Zombie/[2010] Oblivion Mod Falcor p.1/01 - Oblivion Mod Falcor p.1.mp3": "a3c01f164eeca4541aeed49264d2fc8c",
"Project Zombie/[2010] Oblivion Mod Falcor p.1/folder.jpg": "fb95b510681676e81c321171fc23143e"
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
{
".ytdl-sub-split_by_chapters_with_regex_video-download-archive.json": "cb82ae5ed9d7a41aea15446254c93738",
".ytdl-sub-split_by_chapters_with_regex_video-download-archive.json": "9798e8289742586d0efd295a97c6c906",
"Alfa Mist/[2017] Nocturne/01 - Intro (Feat. Racheal Ofori & Barney Artist).mp3": "7be57c4dde9ba2c3fa72e9cc65b0f586",
"Alfa Mist/[2017] Nocturne/02 - Answers (Feat. Rick David & Kaya Thomas - Dyke).mp3": "5ee032da9965c51913fcfa0319d061bd",
"Alfa Mist/[2017] Nocturne/03 - Blaze (Feat. Kaya Thomas - Dyke).mp3": "82c1f35bdccfb4ec146f33661ac94d6b",
"Alfa Mist/[2017] Nocturne/04 - What If (Interlude).mp3": "040b38c249e478d8832dcea8bfeca17f",
"Alfa Mist/[2017] Nocturne/05 - No Peace (Feat. Tom Misch).mp3": "06f7e57fecdc7b4d2e37c7652791ce19",
"Alfa Mist/[2017] Nocturne/06 - Closer (Feat. Lester Duval).mp3": "91a03449c33bf9dcff5a9654a8525626",
"Alfa Mist/[2017] Nocturne/07 - Delusions: Rumination (Interlude) (Feat. Racheal Ofori).mp3": "54c26621b8d4e74f37217c836479a077",
"Alfa Mist/[2017] Nocturne/07 - Delusions Rumination (Interlude) (Feat. Racheal Ofori).mp3": "54c26621b8d4e74f37217c836479a077",
"Alfa Mist/[2017] Nocturne/08 - Dreams (Feat. Carmody).mp3": "2704327ac5998086e77a77da164e2afd",
"Alfa Mist/[2017] Nocturne/09 - Dreaming (Interlude) (Feat. Racheal Ofori).mp3": "75b8a26d3099aa2d9ba488b33c4eb1e7",
"Alfa Mist/[2017] Nocturne/10 - Hopeful (Feat. Jordan Rakei).mp3": "0bf0979c99c55ef986efaba55c0dc858",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
".ytdl-sub-single_song_test-download-archive.json": "22c1431e35033d777a674c4802bcc786",
"YouTube/[2019] YouTube Rewind 2019: For the Record | #YouTubeRewind/01 - YouTube Rewind 2019: For the Record | #YouTubeRewind.mp3": "7affc0ecf01f36de9cee1a7aafd776eb",
"YouTube/[2019] YouTube Rewind 2019: For the Record | #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a"
".ytdl-sub-single_song_test-download-archive.json": "c8ff22ec3304c9f8dab18cedaed4e8b4",
"YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/01 - YouTube Rewind 2019 For the Record #YouTubeRewind.mp3": "7affc0ecf01f36de9cee1a7aafd776eb",
"YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind/folder.jpg": "50ee47c80f679029f5d3503bb91b045a"
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
".ytdl-sub-sponsorblock_with_embedded_subs_test-download-archive.json": "40abc64a8fe93a10406bff1548a2cbe2",
"JMC/JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case-thumb.jpg": "b5353a824a4800cc26f884e3025ed969",
"JMC/JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.info.json": "4304c960965fa4bcf91a3b0f0d97ce62",
"JMC/JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "8ed30bf3716a5f86eff0de78582cff2e",
"JMC/JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.info.json": "e0063c9893bdbb08ac8766c67952be44",
"JMC/JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "d0d1ac72d6a968bb4fe77e79abc784c7",
"JMC/JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo": "b9bd35e4f260c728774d8dd31f83637c"
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
".ytdl-sub-subtitles_embedded_test-download-archive.json": "1aa35c1d2663ac721f33f8374c00af1a",
"JMC/JMC - YouTube Rewind 2019: For the Record | #YouTubeRewind-thumb.jpg": "50ee47c80f679029f5d3503bb91b045a",
"JMC/JMC - YouTube Rewind 2019: For the Record | #YouTubeRewind.info.json": "c7a301a9429360ad4b82c280142c9677",
"JMC/JMC - YouTube Rewind 2019: For the Record | #YouTubeRewind.info.json": "4efbf3e5ff5ceb8589dab87f9934927f",
"JMC/JMC - YouTube Rewind 2019: For the Record | #YouTubeRewind.mp4": "1f1f91f85b162c20596a2413e704b809",
"JMC/JMC - YouTube Rewind 2019: For the Record | #YouTubeRewind.nfo": "c64964fab07574080e5da3242e3bfd48"
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"JMC/JMC - YouTube Rewind 2019: For the Record | #YouTubeRewind-thumb.jpg": "50ee47c80f679029f5d3503bb91b045a",
"JMC/JMC - YouTube Rewind 2019: For the Record | #YouTubeRewind.de.srt": "b343c3bb9257b7ee7ba38f570a115b37",
"JMC/JMC - YouTube Rewind 2019: For the Record | #YouTubeRewind.en.srt": "fe8c6ee92cae6e059fd80fd61691adbe",
"JMC/JMC - YouTube Rewind 2019: For the Record | #YouTubeRewind.info.json": "600bcb2d0040f98a0ee359b3b721be01",
"JMC/JMC - YouTube Rewind 2019: For the Record | #YouTubeRewind.info.json": "3f12b8c905cbffef573ba09ea96ae2aa",
"JMC/JMC - YouTube Rewind 2019: For the Record | #YouTubeRewind.mp4": "1f1f91f85b162c20596a2413e704b809",
"JMC/JMC - YouTube Rewind 2019: For the Record | #YouTubeRewind.nfo": "c64964fab07574080e5da3242e3bfd48"
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"JMC/JMC - Oblivion Mod "Falcor" p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e",
"JMC/JMC - Oblivion Mod "Falcor" p.1.info.json": "99db941978a6972f405d6953ed9784a3",
"JMC/JMC - Oblivion Mod "Falcor" p.1.info.json": "060a3cdff75379aad478eb056d3abaaf",
"JMC/JMC - Oblivion Mod "Falcor" p.1.mp4": "3744c49f2e447bd7712a5aad5ed36be2",
"JMC/JMC - Oblivion Mod "Falcor" p.1.nfo": "24cc4e17d2bebc89b2759ce5471d403e"
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"JMC/JMC - Oblivion Mod "Falcor" p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e",
"JMC/JMC - Oblivion Mod "Falcor" p.1.info.json": "b55f55e7a4251ddd9f3245e65ad175a9",
"JMC/JMC - Oblivion Mod "Falcor" p.1.info.json": "61617f44c5a9bb25fb446f0ce5ea5c35",
"JMC/JMC - Oblivion Mod "Falcor" p.1.mp4": "3744c49f2e447bd7712a5aad5ed36be2",
"JMC/JMC - Oblivion Mod "Falcor" p.1.nfo": "24cc4e17d2bebc89b2759ce5471d403e"
}
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ Files created:
track: 6
tracktotal: 11
year: 2017
07 - 07. Delusions: Rumination (Interlude) (Feat. Racheal Ofori).mp3
07 - 07. Delusions Rumination (Interlude) (Feat. Racheal Ofori).mp3
From Chapter Split:
Warning: Dry-run assumes embedded chapters with no modifications
Source Title: Alfa Mist - Nocturne [Full Album]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ Files created:
track: 6
tracktotal: 11
year: 2017
07 - 07. Delusions: Rumination (Interlude) (Feat. Racheal Ofori).mp3
07 - 07. Delusions Rumination (Interlude) (Feat. Racheal Ofori).mp3
From Chapter Split:
Source Title: Alfa Mist - Nocturne [Full Album]
Segment: 17:57 - 20:05
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ Files created:
----------------------------------------
{output_directory}
.ytdl-sub-split_by_chapters_with_regex_video-download-archive.json
{output_directory}/Project Zombie/[2010] Oblivion Mod "Falcor" p.1
01 - Oblivion Mod "Falcor" p.1.mp3
{output_directory}/Project Zombie/[2010] Oblivion Mod Falcor p.1
01 - Oblivion Mod Falcor p.1.mp3
Music Tags:
album: Oblivion Mod "Falcor" p.1
albumartist: Project Zombie
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ Files created:
track: 6
tracktotal: 11
year: 2017
07 - Delusions: Rumination (Interlude) (Feat. Racheal Ofori).mp3
07 - Delusions Rumination (Interlude) (Feat. Racheal Ofori).mp3
From Chapter Split:
Warning: Dry-run assumes embedded chapters with no modifications
Source Title: Alfa Mist - Nocturne [Full Album]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ Files created:
track: 6
tracktotal: 11
year: 2017
07 - Delusions: Rumination (Interlude) (Feat. Racheal Ofori).mp3
07 - Delusions Rumination (Interlude) (Feat. Racheal Ofori).mp3
From Chapter Split:
Source Title: Alfa Mist - Nocturne [Full Album]
Segment: 17:57 - 20:05
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ Files created:
----------------------------------------
{output_directory}
.ytdl-sub-single_song_test-download-archive.json
{output_directory}/YouTube/[2019] YouTube Rewind 2019: For the Record | #YouTubeRewind
01 - YouTube Rewind 2019: For the Record | #YouTubeRewind.mp3
{output_directory}/YouTube/[2019] YouTube Rewind 2019 For the Record #YouTubeRewind
01 - YouTube Rewind 2019 For the Record #YouTubeRewind.mp3
Embedded Thumbnail, Music Tags:
album: YouTube Rewind 2019: For the Record | #YouTubeRewind
albumartist: YouTube
Expand Down
14 changes: 13 additions & 1 deletion tests/unit/validators/test_string_formatter_validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,14 +153,26 @@ def test_entry_formatter_override_sanitized_recursive(self, string_formatter_cla
variable_dict = {
"level_a": "level a",
"level_b": "level b ? {level_a}",
"level_c_sanitized": "level c and {level_b}",
"level_c": "level c and {level_b}",
}

format_string = string_formatter_class(name="test", value="level d and {level_c_sanitized}")
expected_string = "level d and " + sanitize_filename("level c and level b ? level a")

assert format_string.apply_formatter(variable_dict=variable_dict) == expected_string

def test_entry_formatter_override_sanitized_recursive_inner(self, string_formatter_class):
variable_dict = {
"level_a": "level a ?",
"level_b": "level b ? {level_a_sanitized}",
"level_c": "level c and {level_b_sanitized}",
}

format_string = string_formatter_class(name="test", value="level d and {level_c}")
expected_string = "level d and level c and " + sanitize_filename("level b ? level a ?")

assert format_string.apply_formatter(variable_dict=variable_dict) == expected_string

def test_entry_formatter_override_recursive_fail_cycle(self, string_formatter_class):
variable_dict = {
"level_a": "{level_b}",
Expand Down

0 comments on commit a2808fa

Please sign in to comment.