Skip to content

Commit

Permalink
Return url for file in GCP public bucket instead of error (#754)
Browse files Browse the repository at this point in the history
  • Loading branch information
dreadatour authored Dec 27, 2024
1 parent cf05881 commit c996e50
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 19 deletions.
15 changes: 8 additions & 7 deletions src/datachain/client/gcs.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,14 @@ def create_fs(cls, **kwargs) -> GCSFileSystem:
return cast(GCSFileSystem, super().create_fs(**kwargs))

def url(self, path: str, expires: int = 3600, **kwargs) -> str:
try:
return self.fs.sign(self.get_full_path(path), expiration=expires, **kwargs)
except AttributeError as exc:
is_anon = self.fs.storage_options.get("token") == "anon"
if is_anon and "you need a private key to sign credentials" in str(exc):
return f"https://storage.googleapis.com/{self.name}/{path}"
raise
"""
Generate a signed URL for the given path.
If the client is anonymous, a public URL is returned instead
(see https://cloud.google.com/storage/docs/access-public-data#api-link).
"""
if self.fs.storage_options.get("token") == "anon":
return f"https://storage.googleapis.com/{self.name}/{path}"
return self.fs.sign(self.get_full_path(path), expiration=expires, **kwargs)

@staticmethod
def parse_timestamp(timestamp: str) -> datetime:
Expand Down
13 changes: 1 addition & 12 deletions tests/unit/test_client_gcs.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,6 @@
from datachain.client import Client


def test_anon_url(mocker):
def sign(*args, **kwargs):
raise AttributeError(
"you need a private key to sign credentials."
"the credentials you are currently using"
" <class 'google.oauth2.credentials.Credentials'> just contains a token."
" see https://googleapis.dev/python/google-api-core/latest/auth.html"
"#setting-up-a-service-account for more details."
)

mocker.patch("gcsfs.GCSFileSystem.sign", side_effect=sign)

def test_anon_url():
client = Client.get_client("gs://foo", None, anon=True)
assert client.url("bar") == "https://storage.googleapis.com/foo/bar"

0 comments on commit c996e50

Please sign in to comment.