From 8caa7e8ca8da3be71beb4431b4b55559880571f1 Mon Sep 17 00:00:00 2001 From: Chris Marais Date: Mon, 4 Nov 2024 16:50:20 +0000 Subject: [PATCH 1/2] add timeout support --- cloudpathlib/gs/gsclient.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cloudpathlib/gs/gsclient.py b/cloudpathlib/gs/gsclient.py index edd5b88a..efdab3fa 100644 --- a/cloudpathlib/gs/gsclient.py +++ b/cloudpathlib/gs/gsclient.py @@ -45,6 +45,7 @@ def __init__( local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, download_chunks_concurrently_kwargs: Optional[Dict[str, Any]] = None, + timeout: float = None, ): """Class constructor. Sets up a [`Storage Client`](https://googleapis.dev/python/storage/latest/client.html). @@ -85,6 +86,7 @@ def __init__( download_chunks_concurrently_kwargs (Optional[Dict[str, Any]]): Keyword arguments to pass to [`download_chunks_concurrently`](https://cloud.google.com/python/docs/reference/storage/latest/google.cloud.storage.transfer_manager#google_cloud_storage_transfer_manager_download_chunks_concurrently) for sliced parallel downloads; Only available in `google-cloud-storage` version 2.7.0 or later, otherwise ignored and a warning is emitted. + timeout (float): Cloud Storage [timeout value](https://cloud.google.com/python/docs/reference/storage/1.39.0/retry_timeout) """ if application_credentials is None: application_credentials = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") @@ -102,6 +104,10 @@ def __init__( self.client = StorageClient.create_anonymous_client() self.download_chunks_concurrently_kwargs = download_chunks_concurrently_kwargs + self.timeout = timeout + self.blob_kwargs = {} + if self.timeout: + self.blob_kwargs["timeout"] = self.timeout super().__init__( local_cache_dir=local_cache_dir, @@ -128,7 +134,6 @@ def _download_file(self, cloud_path: GSPath, local_path: Union[str, os.PathLike] blob = bucket.get_blob(cloud_path.blob) local_path = Path(local_path) - if transfer_manager is not None and self.download_chunks_concurrently_kwargs is not None: transfer_manager.download_chunks_concurrently( blob, local_path, **self.download_chunks_concurrently_kwargs @@ -139,7 +144,7 @@ def _download_file(self, cloud_path: GSPath, local_path: Union[str, os.PathLike] "Ignoring `download_chunks_concurrently_kwargs` for version of google-cloud-storage that does not support them (<2.7.0)." ) - blob.download_to_filename(local_path) + blob.download_to_filename(local_path, **self.blob_kwargs) return local_path @@ -246,7 +251,7 @@ def _move_file(self, src: GSPath, dst: GSPath, remove_src: bool = True) -> GSPat dst_bucket = self.client.bucket(dst.bucket) src_blob = src_bucket.get_blob(src.blob) - src_bucket.copy_blob(src_blob, dst_bucket, dst.blob) + src_bucket.copy_blob(src_blob, dst_bucket, dst.blob, **self.blob_kwargs) if remove_src: src_blob.delete() @@ -279,7 +284,7 @@ def _upload_file(self, local_path: Union[str, os.PathLike], cloud_path: GSPath) content_type, _ = self.content_type_method(str(local_path)) extra_args["content_type"] = content_type - blob.upload_from_filename(str(local_path), **extra_args) + blob.upload_from_filename(str(local_path), **extra_args, **self.blob_kwargs) return cloud_path def _get_public_url(self, cloud_path: GSPath) -> str: From 9b034b781af40c664db835cc704d4cc2a6211f76 Mon Sep 17 00:00:00 2001 From: Chris Marais Date: Wed, 13 Nov 2024 11:32:06 +0000 Subject: [PATCH 2/2] no implicit optional --- cloudpathlib/gs/gsclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudpathlib/gs/gsclient.py b/cloudpathlib/gs/gsclient.py index efdab3fa..9fae26f4 100644 --- a/cloudpathlib/gs/gsclient.py +++ b/cloudpathlib/gs/gsclient.py @@ -45,7 +45,7 @@ def __init__( local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, download_chunks_concurrently_kwargs: Optional[Dict[str, Any]] = None, - timeout: float = None, + timeout: Optional[float] = None, ): """Class constructor. Sets up a [`Storage Client`](https://googleapis.dev/python/storage/latest/client.html). @@ -86,7 +86,7 @@ def __init__( download_chunks_concurrently_kwargs (Optional[Dict[str, Any]]): Keyword arguments to pass to [`download_chunks_concurrently`](https://cloud.google.com/python/docs/reference/storage/latest/google.cloud.storage.transfer_manager#google_cloud_storage_transfer_manager_download_chunks_concurrently) for sliced parallel downloads; Only available in `google-cloud-storage` version 2.7.0 or later, otherwise ignored and a warning is emitted. - timeout (float): Cloud Storage [timeout value](https://cloud.google.com/python/docs/reference/storage/1.39.0/retry_timeout) + timeout (Optional[float]): Cloud Storage [timeout value](https://cloud.google.com/python/docs/reference/storage/1.39.0/retry_timeout) """ if application_credentials is None: application_credentials = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")