From d1c24b16e685f269815f81143bb2dcebd78c05ee Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Thu, 15 Feb 2024 08:22:23 -0800 Subject: [PATCH 01/36] Add download functionality to ulwgl_run - When the user does not specify the PROTONPATH, attempt to download the latest Proton if Proton cannot be found in either the Steam compat tools or local cache. Otherwise, prioritize referring to existing ones -- $HOME/.local/share/Steam/compatibilitytools.d and $HOME/.cache/ULWGL. - Effectively, new installations will always download the latest Proton, while existing ones will simply be warned of a later version. Downloading is avoided if the latest version already exists in the cache, and the cache will be used as a last resort to set the variable. When we're unable to find an existing Proton or download one, we raise an error. --- ulwgl_dl_util.py | 178 +++++++++++++++++++++++++++++++++++++++++++++++ ulwgl_run.py | 14 +++- 2 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 ulwgl_dl_util.py diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py new file mode 100644 index 000000000..36029ceb5 --- /dev/null +++ b/ulwgl_dl_util.py @@ -0,0 +1,178 @@ +from pathlib import Path +from os import environ +from requests import get +from tarfile import open as tar_open +from requests import Response +from typing import Dict, List, Tuple, Any, Union +from hashlib import sha512 + + +def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: + """Attempt to find Proton and downloads the latest if PROTONPATH is not set. + + Only fetches the latest if not first found in the Steam compat + Cache is only referred to last + """ + # TODO: Put this in the background + files: List[Tuple[str, str]] = _fetch_releases() + cache: Path = Path(Path().home().as_posix() + "/.cache/ULWGL") + steam_compat: Path = Path( + Path().home().as_posix() + "/.local/share/Steam/compatibilitytools.d" + ) + + cache.mkdir(exist_ok=True, parents=True) + steam_compat.mkdir(exist_ok=True, parents=True) + + # Prioritize the Steam compat + for proton in steam_compat.glob("GE-Proton*"): + print(f"{proton.name} found in: {steam_compat.as_posix()}") + environ["PROTONPATH"] = proton.as_posix() + env["PROTONPATH"] = environ["PROTONPATH"] + + # Notify the user that they're not using the latest + if len(files) == 2 and proton.name != files[1][0][: files[1][0].find(".")]: + print( + "GE-Proton is outdated and requires manual intervention.\nFor latest release, please download " + + files[1][0] + ) + + return env + + # Check if the latest isn't already in the cache + # Assumes the tarball is legitimate + if ( + files + and Path(Path().home().as_posix() + "/.cache/ULWGL" + files[1][0]).is_file() + ): + _extract_dir( + Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(files[1][0]), + steam_compat, + ) + + environ["PROTONPATH"] = steam_compat.joinpath( + proton.name[: proton.name.find(".")] + ).as_posix() + env["PROTONPATH"] = environ["PROTONPATH"] + + # Download the latest if GE-Proton is not in Steam compat + # If the digests mismatched, refer to the cache in the next block + if files: + try: + print("Fetching latest release ...") + _fetch_proton(env, steam_compat, cache, files) + env["PROTONPATH"] = environ["PROTONPATH"] + + return env + except ValueError as err: + print(err) + + # Cache + for proton in cache.glob("GE-Proton*.tar.gz"): + print(f"{proton.name} found in: {cache.as_posix()}") + + # Extract it to Steam compat then set it as Proton + _extract_dir(proton, steam_compat) + + environ["PROTONPATH"] = steam_compat.joinpath( + proton.name[: proton.name.find(".")] + ).as_posix() + env["PROTONPATH"] = environ["PROTONPATH"] + + return env + + # No internet and cache/compat tool is empty, just return and raise an exception from the caller + return env + + +def _fetch_releases() -> List[Tuple[str, str]]: + """Fetch the latest releases from the Github API.""" + resp: Response = get( + "https://api.github.com/repos/GloriousEggroll/proton-ge-custom/releases" + ) + # The file name and its URL as one element + # Checksum will be the first element, GE-Proton second + files: List[Tuple[str, str]] = [] + + if not resp or not resp.status_code == 200: + return files + + # Attempt to acquire the tarball and checksum from the JSON data + releases: List[Dict[str, Any]] = resp.json() + for release in releases: + if "assets" in release: + assets: List[Dict[str, Any]] = release["assets"] + + for asset in assets: + if ( + "name" in asset + and ( + asset["name"].endswith("sum") + or ( + asset["name"].endswith("tar.gz") + and asset["name"].startswith("GE-Proton") + ) + ) + and "browser_download_url" in asset + ): + if asset["name"].endswith("sum"): + files.append((asset["name"], asset["browser_download_url"])) + else: + files.append((asset["name"], asset["browser_download_url"])) + + if len(files) == 2: + break + break + + return files + + +def _fetch_proton( + env: Dict[str, str], steam_compat: Path, cache: Path, files: List[Tuple[str, str]] +) -> Dict[str, str]: + """Download the latest ULWGL-Proton and set it as PROTONPATH.""" + hash, hash_url = files[0] + proton, proton_url = files[1] + stored_digest: str = "" + + # TODO: Parallelize this + print(f"Downloading {hash} ...") + resp_hash: Response = get(hash_url) + print(f"Downloading {proton} ...") + resp: Response = get(proton_url) + if ( + not resp_hash + and resp_hash.status_code != 200 + and not resp + and resp.status_code != 200 + ): + err: str = "Failed.\nFalling back to cache directory ..." + raise ValueError(err) + + print("Completed.") + + # Download the hash + with Path(f"{cache.as_posix()}/{hash}").open(mode="wb") as file: + file.write(resp_hash.content) + stored_digest = Path(f"{cache.as_posix()}/{hash}").read_text().split(" ")[0] + + # If checksum fails, raise an error and fallback to the cache + with Path(f"{cache.as_posix()}/{proton}").open(mode="wb") as file: + file.write(resp.content) + + if sha512(resp.content).hexdigest() != stored_digest: + err: str = "Digests mismatched.\nFalling back to the cache ..." + raise ValueError(err) + print(f"{proton}: SHA512 is OK") + + _extract_dir(Path(f"{cache.as_posix()}/{proton}"), steam_compat) + environ["PROTONPATH"] = steam_compat.joinpath(proton[: proton.find(".")]).as_posix() + env["PROTONPATH"] = environ["PROTONPATH"] + + return env + + +def _extract_dir(proton: Path, steam_compat: Path) -> None: + """Extract from the cache and to another location.""" + with tar_open(proton.as_posix(), "r:gz") as tar: + print(f"Extracting {proton} -> {steam_compat.as_posix()} ...") + tar.extractall(path=steam_compat.as_posix()) diff --git a/ulwgl_run.py b/ulwgl_run.py index d8484f61f..bbc99d720 100755 --- a/ulwgl_run.py +++ b/ulwgl_run.py @@ -10,6 +10,7 @@ import ulwgl_plugins from re import match import subprocess +import ulwgl_dl_util def parse_args() -> Union[Namespace, Tuple[str, List[str]]]: # noqa: D103 @@ -118,9 +119,16 @@ def check_env( "PROTONPATH" not in os.environ or not Path(os.environ["PROTONPATH"]).expanduser().is_dir() ): - err: str = "Environment variable not set or not a directory: PROTONPATH" - raise ValueError(err) - env["PROTONPATH"] = os.environ["PROTONPATH"] + # Attempt to auto set this env var for the user + os.environ["PROTONPATH"] = "" + ulwgl_dl_util.get_ulwgl_proton(env) + else: + env["PROTONPATH"] = os.environ["PROTONPATH"] + + # If download fails/doesn't exist in the system, raise an error + if not os.environ["PROTONPATH"]: + err: str = "GE-Proton could not be found in cache or compatibilitytools.d\nGE-Proton also failed to be downloaded\nPlease set a Proton directory or visit https://github.com/GloriousEggroll/proton-ge-custom/releases" + raise FileNotFoundError(err) return env From 9a116c6a0a16b6bce340ed89892a88e0ca5cd6bc Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Thu, 15 Feb 2024 11:02:23 -0800 Subject: [PATCH 02/36] ulwgl_dl_util: fix bug when referring to cache for latest - Add missing return statement - Delete property access of name --- ulwgl_dl_util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 36029ceb5..4ce306bae 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -44,16 +44,20 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: files and Path(Path().home().as_posix() + "/.cache/ULWGL" + files[1][0]).is_file() ): + proton: str = files[1][0] + print(f"{proton} found in: {cache.as_posix()}") _extract_dir( Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(files[1][0]), steam_compat, ) environ["PROTONPATH"] = steam_compat.joinpath( - proton.name[: proton.name.find(".")] + proton[: proton.find(".")] ).as_posix() env["PROTONPATH"] = environ["PROTONPATH"] + return env + # Download the latest if GE-Proton is not in Steam compat # If the digests mismatched, refer to the cache in the next block if files: From 621b6c1dd59369f2170469ba179bc8d6cf70f7d4 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Thu, 15 Feb 2024 11:03:22 -0800 Subject: [PATCH 03/36] ulwgl_dl_util: prefer joining Paths --- ulwgl_dl_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 4ce306bae..0d9bd1b68 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -41,13 +41,13 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: # Check if the latest isn't already in the cache # Assumes the tarball is legitimate if ( - files - and Path(Path().home().as_posix() + "/.cache/ULWGL" + files[1][0]).is_file() - ): + files and Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(files[1][0]) + ).is_file(): proton: str = files[1][0] + print(f"{proton} found in: {cache.as_posix()}") _extract_dir( - Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(files[1][0]), + Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(proton), steam_compat, ) From 2512cdc23ada190519da8a42e990e0e8f3b3f985 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Thu, 15 Feb 2024 19:39:24 -0800 Subject: [PATCH 04/36] ulwgl_dl_util: add post cleanup routine - When we're downloading the latest Proton, it's possible the user can interrupt download or extraction process which can lead to corrupted or incomplete files. As a result, in the next run of the launcher, a FileNotFoundError can be falsely raised for the $PROTONPATH/proton file. In the case of an interrupt, be sure to remove the relevant files before we exit. --- ulwgl_dl_util.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 0d9bd1b68..e625b0c2a 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -5,6 +5,7 @@ from requests import Response from typing import Dict, List, Tuple, Any, Union from hashlib import sha512 +from shutil import rmtree def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: @@ -43,16 +44,15 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: if ( files and Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(files[1][0]) ).is_file(): - proton: str = files[1][0] - - print(f"{proton} found in: {cache.as_posix()}") + print(files[1][0] + " found in: " + cache.as_posix()) _extract_dir( - Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(proton), + Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(files[1][0]), steam_compat, ) + # Set PROTONPATH to .local/share/Steam/GE-Proton* environ["PROTONPATH"] = steam_compat.joinpath( - proton[: proton.find(".")] + files[1][0][: files[1][0].find(".")] ).as_posix() env["PROTONPATH"] = environ["PROTONPATH"] @@ -68,7 +68,22 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: return env except ValueError as err: + # Digest mismatched, which should be rare + # In this case, just leave it up to the user to handle it and refer to the cache again print(err) + except KeyboardInterrupt: + tarball: str = files[1][0] + proton: str = tarball[: tarball.find(".")] + + # The files may have been left in an incomplete state + # Remove the tarball and proton we had extracted + print("Keyboard Interrupt received.\nCleaning ...") + if cache.joinpath(tarball).is_file(): + print(f"Purging {tarball} in {cache} ...") + cache.joinpath(tarball).unlink() + if steam_compat.joinpath(proton).is_dir(): + print(f"Purging {proton} in {steam_compat} ...") + rmtree(steam_compat.joinpath(proton).as_posix()) # Cache for proton in cache.glob("GE-Proton*.tar.gz"): From ba54f167f48aff80a22ba7ce6edb59a066587c12 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Thu, 15 Feb 2024 23:42:27 -0800 Subject: [PATCH 05/36] ulwgl_dl_util: handle interrupts whenever extracting - When extracting from the cache to the compatibilitytools.d directory, be sure to remove the leftover data after the interrupt to avoid future errors. --- ulwgl_dl_util.py | 102 ++++++++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 37 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index e625b0c2a..b32962908 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -44,60 +44,73 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: if ( files and Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(files[1][0]) ).is_file(): + tarball: str = files[1][0] + proton: str = files[1][0][: files[1][0].find(".")] + print(files[1][0] + " found in: " + cache.as_posix()) - _extract_dir( - Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(files[1][0]), - steam_compat, - ) - - # Set PROTONPATH to .local/share/Steam/GE-Proton* - environ["PROTONPATH"] = steam_compat.joinpath( - files[1][0][: files[1][0].find(".")] - ).as_posix() - env["PROTONPATH"] = environ["PROTONPATH"] + try: + _extract_dir( + Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(tarball), + steam_compat, + ) - return env + # Set PROTONPATH to .local/share/Steam/compatibilitytools.d/GE-Proton* + environ["PROTONPATH"] = steam_compat.joinpath(proton).as_posix() + env["PROTONPATH"] = environ["PROTONPATH"] + + return env + except KeyboardInterrupt: + # Exit cleanly + # Clean up incompleted files/dir + if steam_compat.joinpath(proton).is_dir(): + print(f"Purging {proton} in {steam_compat} ...") + rmtree(steam_compat.joinpath(proton).as_posix()) + + raise # Download the latest if GE-Proton is not in Steam compat # If the digests mismatched, refer to the cache in the next block if files: + print("Fetching latest release ...") try: - print("Fetching latest release ...") _fetch_proton(env, steam_compat, cache, files) env["PROTONPATH"] = environ["PROTONPATH"] return env - except ValueError as err: - # Digest mismatched, which should be rare - # In this case, just leave it up to the user to handle it and refer to the cache again - print(err) + except ValueError: + # Digest mismatched branch + # In this case, just leave it up to the user to handle it + pass except KeyboardInterrupt: - tarball: str = files[1][0] - proton: str = tarball[: tarball.find(".")] - - # The files may have been left in an incomplete state - # Remove the tarball and proton we had extracted - print("Keyboard Interrupt received.\nCleaning ...") - if cache.joinpath(tarball).is_file(): - print(f"Purging {tarball} in {cache} ...") - cache.joinpath(tarball).unlink() - if steam_compat.joinpath(proton).is_dir(): - print(f"Purging {proton} in {steam_compat} ...") - rmtree(steam_compat.joinpath(proton).as_posix()) + # Exit cleanly + # Clean up incompleted files/dir + _cleanup( + files[1][0], files[1][0][: files[1][0].find(".")], cache, steam_compat + ) # Cache - for proton in cache.glob("GE-Proton*.tar.gz"): - print(f"{proton.name} found in: {cache.as_posix()}") + # This point is reached when digests somehow mismatched or user interrupts the extraction process + for tarball in cache.glob("GE-Proton*.tar.gz"): + # Ignore the mismatched file + if files and tarball == cache.joinpath(files[1][0]): + continue - # Extract it to Steam compat then set it as Proton - _extract_dir(proton, steam_compat) + print(f"{tarball.name} found in: {cache.as_posix()}") + try: + _extract_dir(tarball, steam_compat) + environ["PROTONPATH"] = steam_compat.joinpath( + tarball.name[: tarball.name.find(".")] + ).as_posix() + env["PROTONPATH"] = environ["PROTONPATH"] - environ["PROTONPATH"] = steam_compat.joinpath( - proton.name[: proton.name.find(".")] - ).as_posix() - env["PROTONPATH"] = environ["PROTONPATH"] + return env + except KeyboardInterrupt: + proton: str = tarball.name[: tarball.name.find(".")] - return env + if steam_compat.joinpath(proton).is_dir(): + print(f"Purging {proton} in {steam_compat} ...") + rmtree(steam_compat.joinpath(proton).as_posix()) + raise # No internet and cache/compat tool is empty, just return and raise an exception from the caller return env @@ -195,3 +208,18 @@ def _extract_dir(proton: Path, steam_compat: Path) -> None: with tar_open(proton.as_posix(), "r:gz") as tar: print(f"Extracting {proton} -> {steam_compat.as_posix()} ...") tar.extractall(path=steam_compat.as_posix()) + + +def _cleanup(tarball, proton, cache, steam_compat) -> None: + """Remove files that may have been left in an incomplete state to avoid corruption. + + We want to do this when a download for a new release is interrupted + """ + print("Keyboard Interrupt received.\nCleaning ...") + + if cache.joinpath(tarball).is_file(): + print(f"Purging {tarball} in {cache} ...") + cache.joinpath(tarball).unlink() + if steam_compat.joinpath(proton).is_dir(): + print(f"Purging {proton} in {steam_compat} ...") + rmtree(steam_compat.joinpath(proton).as_posix()) From 08d18203f8627986a5257a84634a7efe3d6e2328 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Thu, 15 Feb 2024 23:45:33 -0800 Subject: [PATCH 06/36] ulwgl_dl_util: handle connection timeout --- ulwgl_dl_util.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index b32962908..32f6e44e1 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -3,6 +3,7 @@ from requests import get from tarfile import open as tar_open from requests import Response +from requests import Timeout from typing import Dict, List, Tuple, Any, Union from hashlib import sha512 from shutil import rmtree @@ -14,8 +15,13 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: Only fetches the latest if not first found in the Steam compat Cache is only referred to last """ - # TODO: Put this in the background - files: List[Tuple[str, str]] = _fetch_releases() + files: List[Tuple[str, str]] = [] + + try: + files = _fetch_releases() + except Timeout: + print("Offline.\nContinuing ...") + cache: Path = Path(Path().home().as_posix() + "/.cache/ULWGL") steam_compat: Path = Path( Path().home().as_posix() + "/.local/share/Steam/compatibilitytools.d" @@ -119,7 +125,8 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: def _fetch_releases() -> List[Tuple[str, str]]: """Fetch the latest releases from the Github API.""" resp: Response = get( - "https://api.github.com/repos/GloriousEggroll/proton-ge-custom/releases" + "https://api.github.com/repos/GloriousEggroll/proton-ge-custom/releases", + timeout=30, ) # The file name and its URL as one element # Checksum will be the first element, GE-Proton second @@ -168,9 +175,9 @@ def _fetch_proton( # TODO: Parallelize this print(f"Downloading {hash} ...") - resp_hash: Response = get(hash_url) + resp_hash: Response = get(hash_url, timeout=30) print(f"Downloading {proton} ...") - resp: Response = get(proton_url) + resp: Response = get(proton_url, timeout=150) if ( not resp_hash and resp_hash.status_code != 200 From f64b473157184d17ff87f7a3fd4cd75f89ae139b Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:32:08 -0800 Subject: [PATCH 07/36] ulwgl_dl_util: move steamcompat to func --- ulwgl_dl_util.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 32f6e44e1..40f7f2535 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -31,18 +31,7 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: steam_compat.mkdir(exist_ok=True, parents=True) # Prioritize the Steam compat - for proton in steam_compat.glob("GE-Proton*"): - print(f"{proton.name} found in: {steam_compat.as_posix()}") - environ["PROTONPATH"] = proton.as_posix() - env["PROTONPATH"] = environ["PROTONPATH"] - - # Notify the user that they're not using the latest - if len(files) == 2 and proton.name != files[1][0][: files[1][0].find(".")]: - print( - "GE-Proton is outdated and requires manual intervention.\nFor latest release, please download " - + files[1][0] - ) - + if _get_from_steamcompat(env, steam_compat, cache, files): return env # Check if the latest isn't already in the cache @@ -230,3 +219,24 @@ def _cleanup(tarball, proton, cache, steam_compat) -> None: if steam_compat.joinpath(proton).is_dir(): print(f"Purging {proton} in {steam_compat} ...") rmtree(steam_compat.joinpath(proton).as_posix()) + + +def _get_from_steamcompat( + env: Dict[str, str], steam_compat: Path, cache: Path, files: List[Tuple[str, str]] +) -> Dict[str, str]: + """Refer to Steam compat folder for any existing Proton directories.""" + for proton in steam_compat.glob("GE-Proton*"): + print(f"{proton.name} found in: {steam_compat.as_posix()}") + environ["PROTONPATH"] = proton.as_posix() + env["PROTONPATH"] = environ["PROTONPATH"] + + # Notify the user that they're not using the latest + if len(files) == 2 and proton.name != files[1][0][: files[1][0].find(".")]: + print( + "GE-Proton is outdated and requires manual intervention.\nFor latest release, please download " + + files[1][1] + ) + + return env + + return None From fbb60288ebd67279476cdcfce37765cccb6b7bae Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:34:25 -0800 Subject: [PATCH 08/36] ulwgl_dl_util: move cache to func --- ulwgl_dl_util.py | 68 ++++++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 40f7f2535..aa883560d 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -34,34 +34,9 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: if _get_from_steamcompat(env, steam_compat, cache, files): return env - # Check if the latest isn't already in the cache - # Assumes the tarball is legitimate - if ( - files and Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(files[1][0]) - ).is_file(): - tarball: str = files[1][0] - proton: str = files[1][0][: files[1][0].find(".")] - - print(files[1][0] + " found in: " + cache.as_posix()) - try: - _extract_dir( - Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(tarball), - steam_compat, - ) - - # Set PROTONPATH to .local/share/Steam/compatibilitytools.d/GE-Proton* - environ["PROTONPATH"] = steam_compat.joinpath(proton).as_posix() - env["PROTONPATH"] = environ["PROTONPATH"] - - return env - except KeyboardInterrupt: - # Exit cleanly - # Clean up incompleted files/dir - if steam_compat.joinpath(proton).is_dir(): - print(f"Purging {proton} in {steam_compat} ...") - rmtree(steam_compat.joinpath(proton).as_posix()) - - raise + # Use the latest Proton in the cache if it exists + if _get_from_cache(env, steam_compat, cache, files, True): + return env # Download the latest if GE-Proton is not in Steam compat # If the digests mismatched, refer to the cache in the next block @@ -240,3 +215,40 @@ def _get_from_steamcompat( return env return None + + +def _get_from_cache( + env: Dict[str, str], + steam_compat: Path, + cache: Path, + files: List[Tuple[str, str]], + latest=True, +) -> Dict[str, str]: + """Refer to ULWGL cache directory. + + Use the latest in the cache when present. Older Proton versions are only referred to when: digests mismatch, user interrupt, or download failure/no internet + """ + if files and latest: + tarball: str = files[1][0] # GE-Proton*.tar.gz + proton: str = tarball[: tarball.find(".")] # GE-Proton\d+\-\d\d + + print(tarball + " found in: " + cache.as_posix()) + try: + _extract_dir( + Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(tarball), + steam_compat, + ) + + # Set PROTONPATH to .local/share/Steam/compatibilitytools.d/GE-Proton* + environ["PROTONPATH"] = steam_compat.joinpath(proton).as_posix() + env["PROTONPATH"] = environ["PROTONPATH"] + + return env + except KeyboardInterrupt: + # Exit cleanly + # Clean up only the extracted data + if steam_compat.joinpath(proton).is_dir(): + print(f"Purging {proton} in {steam_compat} ...") + rmtree(steam_compat.joinpath(proton).as_posix()) + + raise From e3563cb8f330410feac9316531d70ca71c70ce3b Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:37:00 -0800 Subject: [PATCH 09/36] ulwgl_dl_util: move cache to func - Code block in which the digest mismatched, user interrupt or failure to download/no internet. --- ulwgl_dl_util.py | 52 ++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index aa883560d..0bdfb85cd 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -59,28 +59,10 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: ) # Cache - # This point is reached when digests somehow mismatched or user interrupts the extraction process - for tarball in cache.glob("GE-Proton*.tar.gz"): - # Ignore the mismatched file - if files and tarball == cache.joinpath(files[1][0]): - continue - - print(f"{tarball.name} found in: {cache.as_posix()}") - try: - _extract_dir(tarball, steam_compat) - environ["PROTONPATH"] = steam_compat.joinpath( - tarball.name[: tarball.name.find(".")] - ).as_posix() - env["PROTONPATH"] = environ["PROTONPATH"] - - return env - except KeyboardInterrupt: - proton: str = tarball.name[: tarball.name.find(".")] - - if steam_compat.joinpath(proton).is_dir(): - print(f"Purging {proton} in {steam_compat} ...") - rmtree(steam_compat.joinpath(proton).as_posix()) - raise + # Refer to an old version previously installed + # Reached on digest mismatch, user interrupt or download failure/no internet + if _get_from_cache(env, steam_compat, cache, files, False): + return env # No internet and cache/compat tool is empty, just return and raise an exception from the caller return env @@ -252,3 +234,29 @@ def _get_from_cache( rmtree(steam_compat.joinpath(proton).as_posix()) raise + + # Refer to an old version previously installed + # Reached on digest mismatch, user interrupt or download failure/no internet + for tarball in cache.glob("GE-Proton*.tar.gz"): + # Ignore the mismatched file + if files and tarball == cache.joinpath(files[1][0]): + continue + + print(f"{tarball.name} found in: {cache.as_posix()}") + try: + _extract_dir(tarball, steam_compat) + environ["PROTONPATH"] = steam_compat.joinpath( + tarball.name[: tarball.name.find(".")] + ).as_posix() + env["PROTONPATH"] = environ["PROTONPATH"] + + break + except KeyboardInterrupt: + proton: str = tarball.name[: tarball.name.find(".")] + + if steam_compat.joinpath(proton).is_dir(): + print(f"Purging {proton} in {steam_compat} ...") + rmtree(steam_compat.joinpath(proton).as_posix()) + raise + + return env From 4691e26f2361bb8b2bd77316fa3ffb8168b7af9c Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:37:52 -0800 Subject: [PATCH 10/36] ulwgl_dl_util: move download to func --- ulwgl_dl_util.py | 49 ++++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 0bdfb85cd..fdae6a4cd 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -38,25 +38,10 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: if _get_from_cache(env, steam_compat, cache, files, True): return env - # Download the latest if GE-Proton is not in Steam compat + # Download the latest if Proton is not in Steam compat # If the digests mismatched, refer to the cache in the next block - if files: - print("Fetching latest release ...") - try: - _fetch_proton(env, steam_compat, cache, files) - env["PROTONPATH"] = environ["PROTONPATH"] - - return env - except ValueError: - # Digest mismatched branch - # In this case, just leave it up to the user to handle it - pass - except KeyboardInterrupt: - # Exit cleanly - # Clean up incompleted files/dir - _cleanup( - files[1][0], files[1][0][: files[1][0].find(".")], cache, steam_compat - ) + if _get_latest(env, steam_compat, cache, files): + return env # Cache # Refer to an old version previously installed @@ -260,3 +245,31 @@ def _get_from_cache( raise return env + + +def _get_latest( + env: Dict[str, str], steam_compat: Path, cache: Path, files: List[Tuple[str, str]] +) -> Dict[str, str]: + """Download the latest Proton for new installs -- empty cache and Steam compat. + + When the digests mismatched or when interrupted, refer to cache for an old version + """ + if files: + tarball: str = files[1][0] + + print("Fetching latest release ...") + try: + _fetch_proton(env, steam_compat, cache, files) + env["PROTONPATH"] = environ["PROTONPATH"] + except ValueError: + # Digest mismatched or download failed + # Refer to the cache for old version next + return None + except KeyboardInterrupt: + # Exit cleanly + # Clean up extracted data and cache to prevent corruption/errors + # Refer to the cache for old version next + _cleanup(tarball, tarball[: tarball.find(".")], cache, steam_compat) + return None + + return env From f24dbc859196550c5dfed3028b531fee7c241e31 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:38:35 -0800 Subject: [PATCH 11/36] ulwgl_dl_util: print on extraction complete --- ulwgl_dl_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index fdae6a4cd..fb7570e30 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -146,6 +146,7 @@ def _extract_dir(proton: Path, steam_compat: Path) -> None: with tar_open(proton.as_posix(), "r:gz") as tar: print(f"Extracting {proton} -> {steam_compat.as_posix()} ...") tar.extractall(path=steam_compat.as_posix()) + print("Completed.") def _cleanup(tarball, proton, cache, steam_compat) -> None: From 05d7f4d141d19e5701e584c5bdfa53fe499740cc Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:41:55 -0800 Subject: [PATCH 12/36] ulwgl_dl_util: move var inside block --- ulwgl_dl_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index fb7570e30..a30997acd 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -256,8 +256,6 @@ def _get_latest( When the digests mismatched or when interrupted, refer to cache for an old version """ if files: - tarball: str = files[1][0] - print("Fetching latest release ...") try: _fetch_proton(env, steam_compat, cache, files) @@ -267,6 +265,8 @@ def _get_latest( # Refer to the cache for old version next return None except KeyboardInterrupt: + tarball: str = files[1][0] + # Exit cleanly # Clean up extracted data and cache to prevent corruption/errors # Refer to the cache for old version next From 9de0628fb024830553069d520eb22afa496283aa Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Fri, 16 Feb 2024 12:06:30 -0800 Subject: [PATCH 13/36] ulwgl_run: only import the func --- ulwgl_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ulwgl_run.py b/ulwgl_run.py index bbc99d720..984004cd2 100755 --- a/ulwgl_run.py +++ b/ulwgl_run.py @@ -10,7 +10,7 @@ import ulwgl_plugins from re import match import subprocess -import ulwgl_dl_util +from ulwgl_dl_util import get_ulwgl_proton def parse_args() -> Union[Namespace, Tuple[str, List[str]]]: # noqa: D103 @@ -121,7 +121,7 @@ def check_env( ): # Attempt to auto set this env var for the user os.environ["PROTONPATH"] = "" - ulwgl_dl_util.get_ulwgl_proton(env) + get_ulwgl_proton(env) else: env["PROTONPATH"] = os.environ["PROTONPATH"] From f6aa2f31e9f9602bcfd2c9928b4d21a36c9d1d2a Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Fri, 16 Feb 2024 12:08:49 -0800 Subject: [PATCH 14/36] ulwgl_dl_util: update comment --- ulwgl_dl_util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index a30997acd..6fdf4498c 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -10,10 +10,10 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: - """Attempt to find Proton and downloads the latest if PROTONPATH is not set. + """Attempt to find existing Proton from the system or downloads the latest if PROTONPATH is not set. - Only fetches the latest if not first found in the Steam compat - Cache is only referred to last + Only fetches the latest if not first found in .local/share/Steam/compatibilitytools.d + The cache directory, .cache/ULWGL, is referenced next for latest or as fallback """ files: List[Tuple[str, str]] = [] From d45784792098546915a26af091c4323a1162b577 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Fri, 16 Feb 2024 12:12:00 -0800 Subject: [PATCH 15/36] ulwgl_test: update tests for new usage - Update the check_env tests for new usage when PROTONPATH is not set by mocking the callout. --- ulwgl_test.py | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/ulwgl_test.py b/ulwgl_test.py index 75fefdc8d..31a378657 100644 --- a/ulwgl_test.py +++ b/ulwgl_test.py @@ -1037,20 +1037,27 @@ def test_parse_args_config(self): result, Namespace, "Expected a Namespace from parse_arg" ) - def test_env_proton_dir(self): - """Test check_env when $PROTONPATH is not a directory. + def test_env_proton_nodir(self): + """Test check_env when $PROTONPATH is not set on failing to setting it. - An ValueError should occur if the value is not a directory + An FileNotFoundError should be raised when we fail to set PROTONPATH """ - with self.assertRaisesRegex(ValueError, "PROTONPATH"): - os.environ["WINEPREFIX"] = self.test_file - os.environ["GAMEID"] = self.test_file - os.environ["PROTONPATH"] = "./foo" - ulwgl_run.check_env(self.env) - self.assertFalse( - Path(os.environ["PROTONPATH"]).is_dir(), - "Expected PROTONPATH to not be a directory", - ) + result = None + + # Mock getting the Proton + with self.assertRaises(FileNotFoundError): + with patch.object( + ulwgl_run, + "get_ulwgl_proton", + return_value=self.env, + ): + os.environ["WINEPREFIX"] = self.test_file + os.environ["GAMEID"] = self.test_file + result = ulwgl_run.check_env(self.env) + # Mock setting it on success + os.environ["PROTONPATH"] = self.test_file + self.assertTrue(result is self.env, "Expected the same reference") + self.assertFalse(os.environ["PROTONPATH"]) def test_env_wine_dir(self): """Test check_env when $WINEPREFIX is not a directory. @@ -1119,10 +1126,20 @@ def test_env_vars(self): def test_env_vars_proton(self): """Test check_env when setting only $WINEPREFIX and $GAMEID.""" - with self.assertRaisesRegex(ValueError, "PROTONPATH"): + with self.assertRaisesRegex(FileNotFoundError, "Proton"): os.environ["WINEPREFIX"] = self.test_file os.environ["GAMEID"] = self.test_file - ulwgl_run.check_env(self.env) + # Mock getting the Proton + with patch.object( + ulwgl_run, + "get_ulwgl_proton", + return_value=self.env, + ): + os.environ["WINEPREFIX"] = self.test_file + os.environ["GAMEID"] = self.test_file + result = ulwgl_run.check_env(self.env) + self.assertTrue(result is self.env, "Expected the same reference") + self.assertFalse(os.environ["PROTONPATH"]) def test_env_vars_wine(self): """Test check_env when setting only $WINEPREFIX.""" From ecde7f642f500bbe9eb4b53456176520856769b3 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Fri, 16 Feb 2024 18:31:26 -0800 Subject: [PATCH 16/36] ulwgl_dl_util: prefer using stdlib than requests module for network --- ulwgl_dl_util.py | 62 ++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 6fdf4498c..301274530 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -1,12 +1,16 @@ from pathlib import Path from os import environ -from requests import get from tarfile import open as tar_open -from requests import Response from requests import Timeout from typing import Dict, List, Tuple, Any, Union from hashlib import sha512 from shutil import rmtree +from http.client import HTTPSConnection +from http.client import HTTPConnection +from http.client import HTTPResponse +from ssl import create_default_context +from json import loads as loads_json +from urllib.request import urlretrieve def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: @@ -55,19 +59,29 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: def _fetch_releases() -> List[Tuple[str, str]]: """Fetch the latest releases from the Github API.""" - resp: Response = get( - "https://api.github.com/repos/GloriousEggroll/proton-ge-custom/releases", - timeout=30, - ) - # The file name and its URL as one element - # Checksum will be the first element, GE-Proton second files: List[Tuple[str, str]] = [] + resp: HTTPResponse = None + conn: HTTPConnection = HTTPSConnection( + "api.github.com", timeout=30, context=create_default_context() + ) + + conn.request( + "GET", + "/repos/GloriousEggroll/proton-ge-custom/releases", + headers={ + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "", + }, + ) - if not resp or not resp.status_code == 200: + resp = conn.getresponse() + + if resp and resp.status != 200: return files # Attempt to acquire the tarball and checksum from the JSON data - releases: List[Dict[str, Any]] = resp.json() + releases: List[Dict[str, Any]] = loads_json(resp.read().decode("utf-8")) for release in releases: if "assets" in release: assets: List[Dict[str, Any]] = release["assets"] @@ -102,34 +116,20 @@ def _fetch_proton( """Download the latest ULWGL-Proton and set it as PROTONPATH.""" hash, hash_url = files[0] proton, proton_url = files[1] - stored_digest: str = "" # TODO: Parallelize this print(f"Downloading {hash} ...") - resp_hash: Response = get(hash_url, timeout=30) + urlretrieve(hash_url, cache.joinpath(hash).as_posix()) print(f"Downloading {proton} ...") - resp: Response = get(proton_url, timeout=150) - if ( - not resp_hash - and resp_hash.status_code != 200 - and not resp - and resp.status_code != 200 - ): - err: str = "Failed.\nFalling back to cache directory ..." - raise ValueError(err) + urlretrieve(proton_url, cache.joinpath(proton).as_posix()) print("Completed.") - # Download the hash - with Path(f"{cache.as_posix()}/{hash}").open(mode="wb") as file: - file.write(resp_hash.content) - stored_digest = Path(f"{cache.as_posix()}/{hash}").read_text().split(" ")[0] - - # If checksum fails, raise an error and fallback to the cache - with Path(f"{cache.as_posix()}/{proton}").open(mode="wb") as file: - file.write(resp.content) - - if sha512(resp.content).hexdigest() != stored_digest: + with cache.joinpath(proton).open(mode="rb") as file: + if ( + sha512(file.read()).hexdigest() + != cache.joinpath(hash).read_text().split(" ")[0] + ): err: str = "Digests mismatched.\nFalling back to the cache ..." raise ValueError(err) print(f"{proton}: SHA512 is OK") From 83a59dcf1df348ebbb49895e8551d1f0c71db733 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Fri, 16 Feb 2024 18:31:45 -0800 Subject: [PATCH 17/36] ulwgl_dl_util: refactor _get_from_cache --- ulwgl_dl_util.py | 53 ++++++++++++++++-------------------------------- 1 file changed, 17 insertions(+), 36 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 301274530..44d2b1ffa 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -190,62 +190,43 @@ def _get_from_cache( steam_compat: Path, cache: Path, files: List[Tuple[str, str]], - latest=True, + use_latest=True, ) -> Dict[str, str]: """Refer to ULWGL cache directory. Use the latest in the cache when present. Older Proton versions are only referred to when: digests mismatch, user interrupt, or download failure/no internet """ - if files and latest: - tarball: str = files[1][0] # GE-Proton*.tar.gz - proton: str = tarball[: tarball.find(".")] # GE-Proton\d+\-\d\d + path: Path = None + name: str = "" - print(tarball + " found in: " + cache.as_posix()) - try: - _extract_dir( - Path(Path().home().as_posix() + "/.cache/ULWGL").joinpath(tarball), - steam_compat, - ) - - # Set PROTONPATH to .local/share/Steam/compatibilitytools.d/GE-Proton* - environ["PROTONPATH"] = steam_compat.joinpath(proton).as_posix() - env["PROTONPATH"] = environ["PROTONPATH"] - - return env - except KeyboardInterrupt: - # Exit cleanly - # Clean up only the extracted data - if steam_compat.joinpath(proton).is_dir(): - print(f"Purging {proton} in {steam_compat} ...") - rmtree(steam_compat.joinpath(proton).as_posix()) - - raise - - # Refer to an old version previously installed - # Reached on digest mismatch, user interrupt or download failure/no internet for tarball in cache.glob("GE-Proton*.tar.gz"): - # Ignore the mismatched file - if files and tarball == cache.joinpath(files[1][0]): - continue - + if files and tarball == cache.joinpath(files[1][0]) and use_latest: + path = tarball + name = tarball.name + elif not use_latest: + path = tarball + name = tarball.name print(f"{tarball.name} found in: {cache.as_posix()}") + break + + if path: try: - _extract_dir(tarball, steam_compat) + _extract_dir(path, steam_compat) environ["PROTONPATH"] = steam_compat.joinpath( - tarball.name[: tarball.name.find(".")] + name[: name.find(".")] ).as_posix() env["PROTONPATH"] = environ["PROTONPATH"] - break + return env except KeyboardInterrupt: - proton: str = tarball.name[: tarball.name.find(".")] + proton: str = name[: name.find(".")] if steam_compat.joinpath(proton).is_dir(): print(f"Purging {proton} in {steam_compat} ...") rmtree(steam_compat.joinpath(proton).as_posix()) raise - return env + return None def _get_latest( From 3a5b4b10f235dd63eddc3e1a3300bfa32c6eecab Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Fri, 16 Feb 2024 18:32:03 -0800 Subject: [PATCH 18/36] ulwgl_dl_util: prefer joining paths --- ulwgl_dl_util.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 44d2b1ffa..3d0711144 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -26,10 +26,8 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: except Timeout: print("Offline.\nContinuing ...") - cache: Path = Path(Path().home().as_posix() + "/.cache/ULWGL") - steam_compat: Path = Path( - Path().home().as_posix() + "/.local/share/Steam/compatibilitytools.d" - ) + cache: Path = Path.home().joinpath(".cache/ULWGL") + steam_compat: Path = Path.home().joinpath(".local/share/Steam/compatibilitytools.d") cache.mkdir(exist_ok=True, parents=True) steam_compat.mkdir(exist_ok=True, parents=True) @@ -134,7 +132,7 @@ def _fetch_proton( raise ValueError(err) print(f"{proton}: SHA512 is OK") - _extract_dir(Path(f"{cache.as_posix()}/{proton}"), steam_compat) + _extract_dir(cache.joinpath(proton), steam_compat) environ["PROTONPATH"] = steam_compat.joinpath(proton[: proton.find(".")]).as_posix() env["PROTONPATH"] = environ["PROTONPATH"] From 3e5e1e138b7a441eae64857edce529de350bfc13 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Fri, 16 Feb 2024 18:40:00 -0800 Subject: [PATCH 19/36] ulwgl_dl_util: delete references to requests module --- ulwgl_dl_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 3d0711144..58c9eada1 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -1,13 +1,13 @@ from pathlib import Path from os import environ from tarfile import open as tar_open -from requests import Timeout from typing import Dict, List, Tuple, Any, Union from hashlib import sha512 from shutil import rmtree from http.client import HTTPSConnection from http.client import HTTPConnection from http.client import HTTPResponse +from http.client import HTTPException from ssl import create_default_context from json import loads as loads_json from urllib.request import urlretrieve @@ -23,7 +23,7 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: try: files = _fetch_releases() - except Timeout: + except HTTPException: print("Offline.\nContinuing ...") cache: Path = Path.home().joinpath(".cache/ULWGL") From 744cedc6b8107a5e69fe021cfe47e1ddff60c161 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 10:37:12 -0800 Subject: [PATCH 20/36] ulwgl_dl_util: be smarter about imports --- ulwgl_dl_util.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 58c9eada1..d3b87392c 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -4,10 +4,7 @@ from typing import Dict, List, Tuple, Any, Union from hashlib import sha512 from shutil import rmtree -from http.client import HTTPSConnection -from http.client import HTTPConnection -from http.client import HTTPResponse -from http.client import HTTPException +from http.client import HTTPSConnection, HTTPResponse, HTTPException, HTTPConnection from ssl import create_default_context from json import loads as loads_json from urllib.request import urlretrieve From 71eaff0ca52fa7f4a5cf6e9df7d2190509c23bc2 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 12:16:27 -0800 Subject: [PATCH 21/36] ulwgl_dl_util: change GE-Proton -> ULWGL-Proton --- ulwgl_dl_util.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index d3b87392c..5ae81c1dd 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -62,7 +62,7 @@ def _fetch_releases() -> List[Tuple[str, str]]: conn.request( "GET", - "/repos/GloriousEggroll/proton-ge-custom/releases", + "/repos/Open-Wine-Components/ULWGL-Proton/releases", headers={ "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", @@ -88,7 +88,7 @@ def _fetch_releases() -> List[Tuple[str, str]]: asset["name"].endswith("sum") or ( asset["name"].endswith("tar.gz") - and asset["name"].startswith("GE-Proton") + and asset["name"].startswith("ULWGL-Proton") ) ) and "browser_download_url" in asset @@ -164,6 +164,8 @@ def _get_from_steamcompat( ) -> Dict[str, str]: """Refer to Steam compat folder for any existing Proton directories.""" for proton in steam_compat.glob("GE-Proton*"): + + for proton in steam_compat.glob("ULWGL-Proton*"): print(f"{proton.name} found in: {steam_compat.as_posix()}") environ["PROTONPATH"] = proton.as_posix() env["PROTONPATH"] = environ["PROTONPATH"] @@ -171,7 +173,7 @@ def _get_from_steamcompat( # Notify the user that they're not using the latest if len(files) == 2 and proton.name != files[1][0][: files[1][0].find(".")]: print( - "GE-Proton is outdated and requires manual intervention.\nFor latest release, please download " + "ULWGL-Proton is outdated.\nFor latest release, please download " + files[1][1] ) @@ -194,7 +196,7 @@ def _get_from_cache( path: Path = None name: str = "" - for tarball in cache.glob("GE-Proton*.tar.gz"): + for tarball in cache.glob("ULWGL-Proton*.tar.gz"): if files and tarball == cache.joinpath(files[1][0]) and use_latest: path = tarball name = tarball.name From 9f2de76c5aaf4f899957ba7342ff7644ebc15a94 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 12:18:43 -0800 Subject: [PATCH 22/36] ulwgl_dl_util: prefer creating a var for the Proton dir - Referencing the tuple element within the array for the tarball name then parsing it was becoming highly error prone/unreadable. --- ulwgl_dl_util.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 5ae81c1dd..068b573fd 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -111,6 +111,9 @@ def _fetch_proton( """Download the latest ULWGL-Proton and set it as PROTONPATH.""" hash, hash_url = files[0] proton, proton_url = files[1] + proton_dir: str = proton[ + : proton.find(".tar.gz") + ] # Proton dir without suffixes/dashes # TODO: Parallelize this print(f"Downloading {hash} ...") @@ -130,7 +133,7 @@ def _fetch_proton( print(f"{proton}: SHA512 is OK") _extract_dir(cache.joinpath(proton), steam_compat) - environ["PROTONPATH"] = steam_compat.joinpath(proton[: proton.find(".")]).as_posix() + environ["PROTONPATH"] = steam_compat.joinpath(proton_dir).as_posix() env["PROTONPATH"] = environ["PROTONPATH"] return env @@ -163,7 +166,12 @@ def _get_from_steamcompat( env: Dict[str, str], steam_compat: Path, cache: Path, files: List[Tuple[str, str]] ) -> Dict[str, str]: """Refer to Steam compat folder for any existing Proton directories.""" - for proton in steam_compat.glob("GE-Proton*"): + proton_dir: str = "" # Latest Proton + + if len(files) == 2: + proton_dir: str = files[1][0][ + : files[1][0].find(".tar.gz") + ] # Proton dir without suffixes/dashes for proton in steam_compat.glob("ULWGL-Proton*"): print(f"{proton.name} found in: {steam_compat.as_posix()}") @@ -171,7 +179,7 @@ def _get_from_steamcompat( env["PROTONPATH"] = environ["PROTONPATH"] # Notify the user that they're not using the latest - if len(files) == 2 and proton.name != files[1][0][: files[1][0].find(".")]: + if proton_dir and proton.name != proton_dir: print( "ULWGL-Proton is outdated.\nFor latest release, please download " + files[1][1] @@ -207,20 +215,20 @@ def _get_from_cache( break if path: + proton_dir: str = name[ + : name.find(".tar.gz") + ] # Proton dir without suffixes/dashes + try: _extract_dir(path, steam_compat) - environ["PROTONPATH"] = steam_compat.joinpath( - name[: name.find(".")] - ).as_posix() + environ["PROTONPATH"] = steam_compat.joinpath(proton_dir).as_posix() env["PROTONPATH"] = environ["PROTONPATH"] return env except KeyboardInterrupt: - proton: str = name[: name.find(".")] - - if steam_compat.joinpath(proton).is_dir(): - print(f"Purging {proton} in {steam_compat} ...") - rmtree(steam_compat.joinpath(proton).as_posix()) + if steam_compat.joinpath(proton_dir).is_dir(): + print(f"Purging {proton_dir} in {steam_compat} ...") + rmtree(steam_compat.joinpath(proton_dir).as_posix()) raise return None @@ -244,11 +252,14 @@ def _get_latest( return None except KeyboardInterrupt: tarball: str = files[1][0] + proton_dir: str = tarball[ + : tarball.find(".tar.gz") + ] # Proton dir without suffixes/dashes # Exit cleanly # Clean up extracted data and cache to prevent corruption/errors # Refer to the cache for old version next - _cleanup(tarball, tarball[: tarball.find(".")], cache, steam_compat) + _cleanup(tarball, proton_dir, cache, steam_compat) return None return env From f666c8f8d0a899c7b146a5d6689de51daf6dcc16 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 12:29:15 -0800 Subject: [PATCH 23/36] ulwgl_run: change error msg - Prefer a shorter message when download fails or Proton cannot be found, and reference ULWGL-Proton Github repository instead of GE-Proton --- ulwgl_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulwgl_run.py b/ulwgl_run.py index 984004cd2..41a2532b9 100755 --- a/ulwgl_run.py +++ b/ulwgl_run.py @@ -127,7 +127,7 @@ def check_env( # If download fails/doesn't exist in the system, raise an error if not os.environ["PROTONPATH"]: - err: str = "GE-Proton could not be found in cache or compatibilitytools.d\nGE-Proton also failed to be downloaded\nPlease set a Proton directory or visit https://github.com/GloriousEggroll/proton-ge-custom/releases" + err: str = "Download failed.\nProton could not be found in cache or compatibilitytools.d\nPlease set $PROTONPATH or visit https://github.com/Open-Wine-Components/ULWGL-Proton/releases" raise FileNotFoundError(err) return env From 679cbffc2a5ce073c38880e11054578543083520 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 14:56:05 -0800 Subject: [PATCH 24/36] ulwgl_dl_util: format --- ulwgl_dl_util.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 068b573fd..99d876ec9 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -111,9 +111,7 @@ def _fetch_proton( """Download the latest ULWGL-Proton and set it as PROTONPATH.""" hash, hash_url = files[0] proton, proton_url = files[1] - proton_dir: str = proton[ - : proton.find(".tar.gz") - ] # Proton dir without suffixes/dashes + proton_dir: str = proton[: proton.find(".tar.gz")] # Proton dir # TODO: Parallelize this print(f"Downloading {hash} ...") @@ -169,9 +167,7 @@ def _get_from_steamcompat( proton_dir: str = "" # Latest Proton if len(files) == 2: - proton_dir: str = files[1][0][ - : files[1][0].find(".tar.gz") - ] # Proton dir without suffixes/dashes + proton_dir: str = files[1][0][: files[1][0].find(".tar.gz")] # Proton dir for proton in steam_compat.glob("ULWGL-Proton*"): print(f"{proton.name} found in: {steam_compat.as_posix()}") @@ -215,9 +211,7 @@ def _get_from_cache( break if path: - proton_dir: str = name[ - : name.find(".tar.gz") - ] # Proton dir without suffixes/dashes + proton_dir: str = name[: name.find(".tar.gz")] # Proton dir try: _extract_dir(path, steam_compat) @@ -252,9 +246,7 @@ def _get_latest( return None except KeyboardInterrupt: tarball: str = files[1][0] - proton_dir: str = tarball[ - : tarball.find(".tar.gz") - ] # Proton dir without suffixes/dashes + proton_dir: str = tarball[: tarball.find(".tar.gz")] # Proton dir # Exit cleanly # Clean up extracted data and cache to prevent corruption/errors From 3c0534edd235aa93c35b4e2995e30c6dbb268d99 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 14:57:21 -0800 Subject: [PATCH 25/36] ulwgl_test: add test when extracting files --- ulwgl_test.py | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/ulwgl_test.py b/ulwgl_test.py index 31a378657..4bbcd329a 100644 --- a/ulwgl_test.py +++ b/ulwgl_test.py @@ -9,6 +9,8 @@ from shutil import rmtree import re import ulwgl_plugins +import ulwgl_dl_util +import tarfile class TestGameLauncher(unittest.TestCase): @@ -49,6 +51,34 @@ def setUp(self): self.test_file = "./tmp.WMYQiPb9A" # Executable self.test_exe = self.test_file + "/" + "foo" + # Cache + self.test_cache = Path("./tmp.5HYdpddgvs") + # Steam compat dir + self.test_compat = Path("./tmp.ZssGZoiNod") + # ULWGL-Proton dir + self.test_proton_dir = Path("ULWGL-Proton-5HYdpddgvs") + # ULWGL-Proton release + self.test_archive = Path(self.test_cache).joinpath( + f"{self.test_proton_dir}.tar.gz" + ) + + self.test_cache.mkdir(exist_ok=True) + self.test_compat.mkdir(exist_ok=True) + self.test_proton_dir.mkdir(exist_ok=True) + + # Mock the proton file in the dir + self.test_proton_dir.joinpath("proton").touch(exist_ok=True) + + # Mock the release downloaded in the cache: tmp.5HYdpddgvs/ULWGL-Proton-5HYdpddgvs.tar.gz + # Expected directory structure within the archive: + # + # +-- ULWGL-Proton-5HYdpddgvs (root directory) + # | +-- proton (normal file) + with tarfile.open(self.test_archive.as_posix(), "w:gz") as tar: + tar.add( + self.test_proton_dir.as_posix(), arcname=self.test_proton_dir.as_posix() + ) + Path(self.test_file).mkdir(exist_ok=True) Path(self.test_exe).touch() @@ -61,6 +91,52 @@ def tearDown(self): if Path(self.test_file).exists(): rmtree(self.test_file) + if self.test_cache.exists(): + rmtree(self.test_cache.as_posix()) + + if self.test_compat.exists(): + rmtree(self.test_compat.as_posix()) + + if self.test_proton_dir.exists(): + rmtree(self.test_proton_dir.as_posix()) + + + def test_extract_err(self): + """Test _extract_dir when passed a non-gzip compressed archive. + + An error should be raised as we only expect .tar.gz releases + """ + test_archive = self.test_cache.joinpath(f"{self.test_proton_dir}.tar") + # Do not apply compression + with tarfile.open(test_archive.as_posix(), "w") as tar: + tar.add( + self.test_proton_dir.as_posix(), arcname=self.test_proton_dir.as_posix() + ) + + with self.assertRaisesRegex(tarfile.ReadError, "gzip"): + ulwgl_dl_util._extract_dir(test_archive, self.test_compat) + + if test_archive.exists(): + test_archive.unlink() + + def test_extract(self): + """Test _extract_dir. + + An error should not be raised when the Proton release is extracted to the Steam compat dir + """ + result = None + + result = ulwgl_dl_util._extract_dir(self.test_archive, self.test_compat) + self.assertFalse(result, "Expected None after extracting") + self.assertTrue( + self.test_compat.joinpath(self.test_proton_dir).exists(), + "Expected proton dir to exists in compat", + ) + self.assertTrue( + self.test_compat.joinpath(self.test_proton_dir).joinpath("proton").exists(), + "Expected 'proton' file to exists in the proton dir", + ) + def test_game_drive_empty(self): """Test enable_steam_game_drive. From ae6d5675d0a580023936e04c817e9415576d1ed9 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 15:02:45 -0800 Subject: [PATCH 26/36] ulwgl_test: add test when removing files - Arguably the most important test of this test suite as we do **not** want to remove files that isn't ours. We test both the positive and negative case. In the latter, the state of the cache or Steam compat should be left the same state as it was before executing the function when passed files that do not exist. --- ulwgl_test.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/ulwgl_test.py b/ulwgl_test.py index 4bbcd329a..e4a96f2da 100644 --- a/ulwgl_test.py +++ b/ulwgl_test.py @@ -100,6 +100,81 @@ def tearDown(self): if self.test_proton_dir.exists(): rmtree(self.test_proton_dir.as_posix()) + def test_cleanup_no_exists(self): + """Test _cleanup when passed files that do not exist. + + In the event of an interrupt during the download/extract process, we only want to clean the files that exist + NOTE: This is **extremely** important, as we do **not** want to delete anything else but the files we downloaded/extracted -- the incomplete tarball/extracted dir + """ + result = None + + ulwgl_dl_util._extract_dir(self.test_archive, self.test_compat) + + # Before cleaning + # On setUp, an archive is created and a dir should exist in compat after extraction + self.assertTrue( + self.test_compat.joinpath(self.test_proton_dir).exists(), + "Expected proton dir to exist in compat before cleaning", + ) + self.assertTrue( + self.test_archive.exists(), + "Expected archive to exist in cache before cleaning", + ) + self.assertTrue( + self.test_compat.joinpath(self.test_proton_dir).joinpath("proton").exists(), + "Expected 'proton' to exist before cleaning", + ) + + # Pass files that do not exist + result = ulwgl_dl_util._cleanup( + "foo.tar.gz", + "foo", + self.test_cache, + self.test_compat, + ) + + # Verify state of cache and compat after cleaning + self.assertFalse(result, "Expected None after cleaning") + self.assertTrue( + self.test_compat.joinpath(self.test_proton_dir).exists(), + "Expected proton dir to still exist after cleaning", + ) + self.assertTrue( + self.test_archive.exists(), + "Expected archive to still exist after cleaning", + ) + self.assertTrue( + self.test_compat.joinpath(self.test_proton_dir).joinpath("proton").exists(), + "Expected 'proton' to still exist after cleaning", + ) + + def test_cleanup(self): + """Test _cleanup. + + In the event of an interrupt during the download/extract process, we want to clean the cache or the extracted dir in Steam compat to avoid incomplete files + """ + result = None + + ulwgl_dl_util._extract_dir(self.test_archive, self.test_compat) + result = ulwgl_dl_util._cleanup( + self.test_proton_dir.as_posix() + ".tar.gz", + self.test_proton_dir.as_posix(), + self.test_cache, + self.test_compat, + ) + self.assertFalse(result, "Expected None after cleaning") + self.assertFalse( + self.test_compat.joinpath(self.test_proton_dir).exists(), + "Expected proton dir to be cleaned in compat", + ) + self.assertFalse( + self.test_archive.exists(), + "Expected archive to be cleaned in cache", + ) + self.assertFalse( + self.test_compat.joinpath(self.test_proton_dir).joinpath("proton").exists(), + "Expected 'proton' to not exist after cleaned", + ) def test_extract_err(self): """Test _extract_dir when passed a non-gzip compressed archive. From 9a425dd7847b3cfa2000b4dcec0686303da18cf9 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 15:13:23 -0800 Subject: [PATCH 27/36] ulwgl_test: update test when removing files - Creates more test files within the compat/cache directories to verify that there wasn't any side effects after executing _cleanup --- ulwgl_test.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/ulwgl_test.py b/ulwgl_test.py index e4a96f2da..af24c7682 100644 --- a/ulwgl_test.py +++ b/ulwgl_test.py @@ -110,11 +110,19 @@ def test_cleanup_no_exists(self): ulwgl_dl_util._extract_dir(self.test_archive, self.test_compat) + # Create a file in the cache and compat + self.test_cache.joinpath("foo").touch() + self.test_compat.joinpath("foo").touch() + # Before cleaning # On setUp, an archive is created and a dir should exist in compat after extraction self.assertTrue( - self.test_compat.joinpath(self.test_proton_dir).exists(), - "Expected proton dir to exist in compat before cleaning", + self.test_compat.joinpath("foo").exists(), + "Expected test file to exist in compat before cleaning", + ) + self.assertTrue( + self.test_cache.joinpath("foo").exists(), + "Expected test file to exist in cache before cleaning", ) self.assertTrue( self.test_archive.exists(), @@ -135,6 +143,14 @@ def test_cleanup_no_exists(self): # Verify state of cache and compat after cleaning self.assertFalse(result, "Expected None after cleaning") + self.assertTrue( + self.test_compat.joinpath("foo").exists(), + "Expected test file to exist in compat after cleaning", + ) + self.assertTrue( + self.test_cache.joinpath("foo").exists(), + "Expected test file to exist in cache after cleaning", + ) self.assertTrue( self.test_compat.joinpath(self.test_proton_dir).exists(), "Expected proton dir to still exist after cleaning", From 88a47e3ea084f38e0fb9979bf0a7c6ae9611ee2d Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 17:32:58 -0800 Subject: [PATCH 28/36] ulwgl_dl_util: fix bug when referencing old Proton from cache - Fixes a logic error by terminating prematurely when referencing an old Proton version that has been saved in the cache --- ulwgl_dl_util.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 99d876ec9..f2a864c36 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -204,11 +204,13 @@ def _get_from_cache( if files and tarball == cache.joinpath(files[1][0]) and use_latest: path = tarball name = tarball.name - elif not use_latest: + break + if tarball != cache.joinpath(files[1][0]) and not use_latest: path = tarball name = tarball.name - print(f"{tarball.name} found in: {cache.as_posix()}") - break + break + + print(f"{name} found in: {path}") if path: proton_dir: str = name[: name.find(".tar.gz")] # Proton dir From 15c483ed60954d0bfa0e3c0c243bc91d4cc068ca Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 17:34:54 -0800 Subject: [PATCH 29/36] ulwgl_test: add tests for steam compat --- ulwgl_test.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/ulwgl_test.py b/ulwgl_test.py index af24c7682..18c1257d3 100644 --- a/ulwgl_test.py +++ b/ulwgl_test.py @@ -100,6 +100,45 @@ def tearDown(self): if self.test_proton_dir.exists(): rmtree(self.test_proton_dir.as_posix()) + + def test_steamcompat_nodir(self): + """Test _get_from_steamcompat when a Proton doesn't exist in the Steam compat dir. + + In this case, the None should be returned to signal that we should continue with downloading the latest Proton + """ + result = None + files = [("", ""), (self.test_archive.name, "")] + + result = ulwgl_dl_util._get_from_steamcompat( + self.env, self.test_compat, self.test_cache, files + ) + + self.assertFalse(result, "Expected None after calling _get_from_steamcompat") + self.assertFalse(self.env["PROTONPATH"], "Expected PROTONPATH to not be set") + + def test_steamcompat(self): + """Test _get_from_steamcompat. + + When a Proton exist in .local/share/Steam/compatibilitytools.d, use it when PROTONPATH is unset + """ + result = None + files = [("", ""), (self.test_archive.name, "")] + + ulwgl_dl_util._extract_dir(self.test_archive, self.test_compat) + + result = ulwgl_dl_util._get_from_steamcompat( + self.env, self.test_compat, self.test_cache, files + ) + + self.assertTrue(result is self.env, "Expected the same reference") + self.assertEqual( + self.env["PROTONPATH"], + self.test_compat.joinpath( + self.test_archive.name[: self.test_archive.name.find(".tar.gz")] + ).as_posix(), + "Expected PROTONPATH to be proton dir in compat", + ) + def test_cleanup_no_exists(self): """Test _cleanup when passed files that do not exist. From 63fba6c892fa3e5f23198bdd8a719de8a663ea82 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 17:35:16 -0800 Subject: [PATCH 30/36] ulwgl_test: add tests for cache --- ulwgl_test.py | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/ulwgl_test.py b/ulwgl_test.py index 18c1257d3..bbe5eb8d6 100644 --- a/ulwgl_test.py +++ b/ulwgl_test.py @@ -100,6 +100,120 @@ def tearDown(self): if self.test_proton_dir.exists(): rmtree(self.test_proton_dir.as_posix()) + def test_cache_interrupt(self): + """Test _get_from_cache on keyboard interrupt.""" + result = None + # In the real usage, should be populated after successful callout for latest Proton releases + # Just mock it and assumes its the latest + files = [("", ""), (self.test_archive.name, "")] + + # Populate the Steam compat dir by extracting the tarball + ulwgl_dl_util._extract_dir(self.test_archive, self.test_compat) + + self.assertTrue( + self.test_compat.joinpath( + self.test_archive.name[: self.test_archive.name.find(".tar.gz")] + ).exists(), + "Expected Proton dir to exist in compat", + ) + + with patch("ulwgl_dl_util._extract_dir") as mock_function: + with self.assertRaisesRegex(KeyboardInterrupt, ""): + # Mock the interrupt + # We want the dir we tried to extract to be cleaned + mock_function.side_effect = KeyboardInterrupt + result = ulwgl_dl_util._get_from_cache( + self.env, self.test_compat, self.test_cache, files, True + ) + + self.assertFalse(result, "Expected None on keyboard interrupt") + self.assertFalse( + self.env["PROTONPATH"], + "Expected PROTONPATH to be empty when the cache is empty", + ) + self.assertFalse( + self.test_compat.joinpath( + self.test_archive.name[: self.test_archive.name.find(".tar.gz")] + ).exists(), + "Expected Proton dir in compat to be cleaned", + ) + + def test_cache_old(self): + """Test _get_from_cache when the cache is empty. + + In real usage, this only happens as a last resort when: download fails, digests mismatched, etc. + """ + result = None + # In the real usage, should be populated after successful callout for latest Proton releases + # Just mock it and assumes its the latest + files = [("", ""), (self.test_archive.name, "")] + + # Mock old Proton versions in the cache + test_proton_dir = Path("ULWGL-Proton-foo") + test_proton_dir.mkdir(exist_ok=True) + test_archive = Path(self.test_cache).joinpath( + f"{test_proton_dir.as_posix()}.tar.gz" + ) + + with tarfile.open(test_archive.as_posix(), "w:gz") as tar: + tar.add(test_proton_dir.as_posix(), arcname=test_proton_dir.as_posix()) + + result = ulwgl_dl_util._get_from_cache( + self.env, self.test_compat, self.test_cache, files, False + ) + + # Verify that the old Proton was assigned + self.assertTrue(result is self.env, "Expected the same reference") + self.assertEqual( + self.env["PROTONPATH"], + self.test_compat.joinpath( + test_archive.name[: test_archive.name.find(".tar.gz")] + ).as_posix(), + "Expected PROTONPATH to be proton dir in compat", + ) + + test_archive.unlink() + test_proton_dir.rmdir() + + def test_cache_empty(self): + """Test _get_from_cache when the cache is empty.""" + result = None + # In the real usage, should be populated after successful callout for latest Proton releases + # Just mock it and assumes its the latest + files = [("", ""), (self.test_archive.name, "")] + + self.test_archive.unlink() + + result = ulwgl_dl_util._get_from_cache( + self.env, self.test_compat, self.test_cache, files, True + ) + self.assertFalse(result, "Expected None when calling _get_from_cache") + self.assertFalse( + self.env["PROTONPATH"], + "Expected PROTONPATH to be empty when the cache is empty", + ) + + def test_cache(self): + """Test _get_from_cache. + + Tests the case when the latest Proton already exists in the cache + """ + result = None + # In the real usage, should be populated after successful callout for latest Proton releases + # Just mock it and assumes its the latest + files = [("", ""), (self.test_archive.name, "")] + + result = ulwgl_dl_util._get_from_cache( + self.env, self.test_compat, self.test_cache, files, True + ) + self.assertTrue(result is self.env, "Expected the same reference") + self.assertEqual( + self.env["PROTONPATH"], + self.test_compat.joinpath( + self.test_archive.name[: self.test_archive.name.find(".tar.gz")] + ).as_posix(), + "Expected PROTONPATH to be proton dir in compat", + ) def test_steamcompat_nodir(self): """Test _get_from_steamcompat when a Proton doesn't exist in the Steam compat dir. From dfa8f975db8e5f831abf4d3a78c65cb6fd44cceb Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 18:50:19 -0800 Subject: [PATCH 31/36] ulwgl_test: update test when interrupting cache extraction - Removes assertions that check for state of variables when an error is raised and updates comments --- ulwgl_test.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ulwgl_test.py b/ulwgl_test.py index bbe5eb8d6..c305136c2 100644 --- a/ulwgl_test.py +++ b/ulwgl_test.py @@ -104,10 +104,13 @@ def test_cache_interrupt(self): """Test _get_from_cache on keyboard interrupt.""" result = None # In the real usage, should be populated after successful callout for latest Proton releases + + def test_cache_interrupt(self): + """Test _get_from_cache on keyboard interrupt on extraction from the cache to the compat dir.""" + # In the real usage, should be populated after successful callout for latest Proton releases # Just mock it and assumes its the latest files = [("", ""), (self.test_archive.name, "")] - # Populate the Steam compat dir by extracting the tarball ulwgl_dl_util._extract_dir(self.test_archive, self.test_compat) self.assertTrue( @@ -120,17 +123,15 @@ def test_cache_interrupt(self): with patch("ulwgl_dl_util._extract_dir") as mock_function: with self.assertRaisesRegex(KeyboardInterrupt, ""): # Mock the interrupt + # We want to simulate an interrupt mid-extraction in this case # We want the dir we tried to extract to be cleaned mock_function.side_effect = KeyboardInterrupt - result = ulwgl_dl_util._get_from_cache( + ulwgl_dl_util._get_from_cache( self.env, self.test_compat, self.test_cache, files, True ) - self.assertFalse(result, "Expected None on keyboard interrupt") - self.assertFalse( - self.env["PROTONPATH"], - "Expected PROTONPATH to be empty when the cache is empty", - ) + # After interrupt, we attempt to clean the compat dir for the file we tried to extract because it could be in an incomplete state + # Verify that the dir we tried to extract from cache is removed to avoid corruption on next launch self.assertFalse( self.test_compat.joinpath( self.test_archive.name[: self.test_archive.name.find(".tar.gz")] From 8f6d383c9f16cfb378d291fef000f67d2c2058be Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 18:54:42 -0800 Subject: [PATCH 32/36] ulwgl_test: add tests for when retrieving latest Proton release --- ulwgl_test.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/ulwgl_test.py b/ulwgl_test.py index c305136c2..e03190290 100644 --- a/ulwgl_test.py +++ b/ulwgl_test.py @@ -100,10 +100,78 @@ def tearDown(self): if self.test_proton_dir.exists(): rmtree(self.test_proton_dir.as_posix()) - def test_cache_interrupt(self): - """Test _get_from_cache on keyboard interrupt.""" + def test_latest_interrupt(self): + """Test _get_latest in the event the user interrupts the download/extraction process. + + Assumes a file is being downloaded or extracted in this case. + A KeyboardInterrupt should be raised, and the cache/compat dir should be cleaned afterwards. + """ + result = None + # In the real usage, should be populated after successful callout for latest Proton releases + # In this case, assume the test variable will be downloaded + files = [("", ""), (self.test_archive.name, "")] + + # In the event of an interrupt, both the cache/compat dir will be checked for the latest release for removal + # We do this since the extraction process can be interrupted as well + ulwgl_dl_util._extract_dir(self.test_archive, self.test_compat) + + with patch("ulwgl_dl_util._fetch_proton") as mock_function: + # Mock the interrupt + # We want the dir we tried to extract to be cleaned + mock_function.side_effect = KeyboardInterrupt + result = ulwgl_dl_util._get_latest( + self.env, self.test_compat, self.test_cache, files + ) + self.assertFalse(self.env["PROTONPATH"], "Expected PROTONPATH to be empty") + self.assertFalse(result, "Expected None when a ValueError occurs") + + # Verify the state of the compat dir/cache + self.assertFalse( + self.test_compat.joinpath( + self.test_archive.name[: self.test_archive.name.find(".tar.gz")] + ).exists(), + "Expected Proton dir in compat to be cleaned", + ) + self.assertFalse( + self.test_cache.joinpath(self.test_archive.name).exists(), + "Expected Proton dir in compat to be cleaned", + ) + + def test_latest_val_err(self): + """Test _get_latest in the event something goes wrong in the download process for the latest Proton. + + Assumes a file is being downloaded in this case. + A ValueError should be raised, and one case it can happen is if the digests mismatched for some reason + """ + result = None + # In the real usage, should be populated after successful callout for latest Proton releases + # When empty, it means the callout failed for some reason (e.g. no internet) + files = [("", ""), (self.test_archive.name, "")] + + with patch("ulwgl_dl_util._fetch_proton") as mock_function: + # Mock the interrupt + mock_function.side_effect = ValueError + result = ulwgl_dl_util._get_latest( + self.env, self.test_compat, self.test_cache, files + ) + self.assertFalse(self.env["PROTONPATH"], "Expected PROTONPATH to be empty") + self.assertFalse(result, "Expected None when a ValueError occurs") + + def test_latest_offline(self): + """Test _get_latest when the user doesn't have internet.""" result = None # In the real usage, should be populated after successful callout for latest Proton releases + # When empty, it means the callout failed for some reason (e.g. no internet) + files = [] + + os.environ["PROTONPATH"] = "" + + with patch("ulwgl_dl_util._fetch_proton"): + result = ulwgl_dl_util._get_latest( + self.env, self.test_compat, self.test_cache, files + ) + self.assertFalse(self.env["PROTONPATH"], "Expected PROTONPATH to be empty") + self.assertTrue(result is self.env, "Expected the same reference") def test_cache_interrupt(self): """Test _get_from_cache on keyboard interrupt on extraction from the cache to the compat dir.""" From 5e23219c444a483aae573d774bfcf9d87409e3af Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 19:04:07 -0800 Subject: [PATCH 33/36] ulwgl_dl_util: fix bug when printing tarball in cache - In the case the cache is empty ' found in: None' can be printed. --- ulwgl_dl_util.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index f2a864c36..73c060910 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -210,11 +210,10 @@ def _get_from_cache( name = tarball.name break - print(f"{name} found in: {path}") - if path: proton_dir: str = name[: name.find(".tar.gz")] # Proton dir + print(f"{name} found in: {path}") try: _extract_dir(path, steam_compat) environ["PROTONPATH"] = steam_compat.joinpath(proton_dir).as_posix() From 811ea0a8b839bfa68561d36021c105ec2fed6cec Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 19:07:03 -0800 Subject: [PATCH 34/36] ulwgl_dl_util: add types to _cleanup --- ulwgl_dl_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 73c060910..12ffce67b 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -145,7 +145,7 @@ def _extract_dir(proton: Path, steam_compat: Path) -> None: print("Completed.") -def _cleanup(tarball, proton, cache, steam_compat) -> None: +def _cleanup(tarball: str, proton: str, cache: Path, steam_compat: Path) -> None: """Remove files that may have been left in an incomplete state to avoid corruption. We want to do this when a download for a new release is interrupted From 642357280a421e3c4798ef06900d7bfdc7483de7 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 19:07:55 -0800 Subject: [PATCH 35/36] ulwgl_dl_util: update comments --- ulwgl_dl_util.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index 12ffce67b..d8bcd1fc1 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -14,7 +14,7 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: """Attempt to find existing Proton from the system or downloads the latest if PROTONPATH is not set. Only fetches the latest if not first found in .local/share/Steam/compatibilitytools.d - The cache directory, .cache/ULWGL, is referenced next for latest or as fallback + .cache/ULWGL is referenced for the latest then as fallback """ files: List[Tuple[str, str]] = [] @@ -42,8 +42,7 @@ def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: if _get_latest(env, steam_compat, cache, files): return env - # Cache - # Refer to an old version previously installed + # Refer to an old version previously downloaded # Reached on digest mismatch, user interrupt or download failure/no internet if _get_from_cache(env, steam_compat, cache, files, False): return env @@ -126,7 +125,7 @@ def _fetch_proton( sha512(file.read()).hexdigest() != cache.joinpath(hash).read_text().split(" ")[0] ): - err: str = "Digests mismatched.\nFalling back to the cache ..." + err: str = "Digests mismatched.\nFalling back to cache ..." raise ValueError(err) print(f"{proton}: SHA512 is OK") @@ -138,7 +137,7 @@ def _fetch_proton( def _extract_dir(proton: Path, steam_compat: Path) -> None: - """Extract from the cache and to another location.""" + """Extract from the cache to another location.""" with tar_open(proton.as_posix(), "r:gz") as tar: print(f"Extracting {proton} -> {steam_compat.as_posix()} ...") tar.extractall(path=steam_compat.as_posix()) @@ -150,7 +149,7 @@ def _cleanup(tarball: str, proton: str, cache: Path, steam_compat: Path) -> None We want to do this when a download for a new release is interrupted """ - print("Keyboard Interrupt received.\nCleaning ...") + print("Keyboard Interrupt.\nCleaning ...") if cache.joinpath(tarball).is_file(): print(f"Purging {tarball} in {cache} ...") @@ -167,7 +166,7 @@ def _get_from_steamcompat( proton_dir: str = "" # Latest Proton if len(files) == 2: - proton_dir: str = files[1][0][: files[1][0].find(".tar.gz")] # Proton dir + proton_dir: str = files[1][0][: files[1][0].find(".tar.gz")] for proton in steam_compat.glob("ULWGL-Proton*"): print(f"{proton.name} found in: {steam_compat.as_posix()}") @@ -195,7 +194,8 @@ def _get_from_cache( ) -> Dict[str, str]: """Refer to ULWGL cache directory. - Use the latest in the cache when present. Older Proton versions are only referred to when: digests mismatch, user interrupt, or download failure/no internet + Use the latest in the cache when present. When download fails, use an old version + Older Proton versions are only referred to when: digests mismatch, user interrupt, or download failure/no internet """ path: Path = None name: str = "" From 45d8ce39c283d1c46a6416d17b28497c75151993 Mon Sep 17 00:00:00 2001 From: R1kaB3rN <100738684+R1kaB3rN@users.noreply.github.com> Date: Sat, 17 Feb 2024 19:15:29 -0800 Subject: [PATCH 36/36] ulwgl_dl_util: update types --- ulwgl_dl_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ulwgl_dl_util.py b/ulwgl_dl_util.py index d8bcd1fc1..aa13761f6 100644 --- a/ulwgl_dl_util.py +++ b/ulwgl_dl_util.py @@ -10,7 +10,7 @@ from urllib.request import urlretrieve -def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str], None]: +def get_ulwgl_proton(env: Dict[str, str]) -> Union[Dict[str, str]]: """Attempt to find existing Proton from the system or downloads the latest if PROTONPATH is not set. Only fetches the latest if not first found in .local/share/Steam/compatibilitytools.d @@ -161,7 +161,7 @@ def _cleanup(tarball: str, proton: str, cache: Path, steam_compat: Path) -> None def _get_from_steamcompat( env: Dict[str, str], steam_compat: Path, cache: Path, files: List[Tuple[str, str]] -) -> Dict[str, str]: +) -> Union[Dict[str, str], None]: """Refer to Steam compat folder for any existing Proton directories.""" proton_dir: str = "" # Latest Proton @@ -191,7 +191,7 @@ def _get_from_cache( cache: Path, files: List[Tuple[str, str]], use_latest=True, -) -> Dict[str, str]: +) -> Union[Dict[str, str], None]: """Refer to ULWGL cache directory. Use the latest in the cache when present. When download fails, use an old version @@ -231,7 +231,7 @@ def _get_from_cache( def _get_latest( env: Dict[str, str], steam_compat: Path, cache: Path, files: List[Tuple[str, str]] -) -> Dict[str, str]: +) -> Union[Dict[str, str], None]: """Download the latest Proton for new installs -- empty cache and Steam compat. When the digests mismatched or when interrupted, refer to cache for an old version