Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
  • Loading branch information
micafer committed Nov 26, 2024
1 parent 1f16f00 commit 9d1c0e9
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 8 deletions.
12 changes: 8 additions & 4 deletions datahugger/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,14 @@ def _get_attr_link(self, record, **kwargs):
return self._get_attr_attr(record, self.ATTR_FILE_LINK_JSONPATH)

def _get_attr_name(self, record):
if not hasattr(self, "ATTR_NAME_JSONPATH"):
return None

return self._get_attr_attr(record, self.ATTR_NAME_JSONPATH)
# if the name is defined, get the name
if hasattr(self, "ATTR_NAME_JSONPATH"):
return self._get_attr_attr(record, self.ATTR_NAME_JSONPATH)
# if not, try to get the name from the link
elif self._get_attr_link(record):
url = self._get_attr_link(record)
return url.split("/")[-1]
return None

def _get_attr_size(self, record):
if not hasattr(self, "ATTR_SIZE_JSONPATH"):
Expand Down
9 changes: 7 additions & 2 deletions datahugger/services.py
Original file line number Diff line number Diff line change
Expand Up @@ -402,9 +402,14 @@ class DataEuropaDataset(DatasetDownloader):
META_FILES_JSONPATH = '$.@graph[?(@.@type == "dcat:Distribution")]'

# paths to file attributes
ATTR_FILE_LINK_JSONPATH = "'dcat:accessURL'.@id"
ATTR_NAME_JSONPATH = "'dct:title'"
ATTR_SIZE_JSONPATH = "'dcat:byteSize'.@value"
ATTR_FILE_LINK_JSONPATH = "'dcat:downloadURL'.@id"

def _get_attr_link(self, record, **kwargs):
if self._get_attr_attr(record, "'dcat:downloadURL'.@id"):
return self._get_attr_attr(record, "'dcat:downloadURL'.@id")
else:
return self._get_attr_attr(record, "'dcat:accessURL'.@id")


class SeaNoeDataset(DatasetDownloader):
Expand Down
4 changes: 2 additions & 2 deletions tests/test_repositories.toml
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,8 @@ location = "https://b2share.eudat.eu/records/db2ef5890fa44c7a85af366a50de73b9"
files = "2024-02-13.sav"

[[dataeuropa]]
location = "https://data.europa.eu/data/datasets/65e092e4009f18f050b14216"
files = "consolidation-wattzhub-schema-irve-statique-20240220-152202.csv"
location = "https://data.europa.eu/data/datasets/9qee-iv7c"
files = "9qee-iv7c.csv"

[[seanoe]]
location = "https://doi.org/10.17882/101042"
Expand Down

0 comments on commit 9d1c0e9

Please sign in to comment.