merge data_europa_eu

datahugger/base.py

@@ -11,7 +11,7 @@
 from urllib.parse import urlparse
 
 import requests
-from jsonpath_ng import parse
+from jsonpath_ng.ext import parse
 from scitree import scitree
 from tqdm import tqdm
 

datahugger/config.py

@@ -4,6 +4,7 @@
 from datahugger.services import DataverseDataset
 from datahugger.services import DjehutyDataset
 from datahugger.services import DSpaceDataset
+from datahugger.services import DataEuropaDataset
 from datahugger.services import FigShareDataset
 from datahugger.services import GitHubDataset
 from datahugger.services import HuggingFaceDataset
@@ -117,6 +118,7 @@
     "trolling.uit.no": DataverseDataset,
     "www.sodha.be": DataverseDataset,
     "www.uni-hildesheim.de": DataverseDataset,
+    "data.europa.eu": DataEuropaDataset,
 }
 
 # regexp lookup

datahugger/services.py

@@ -408,3 +408,20 @@ class SeaNoeDataset(DatasetDownloader):
     ATTR_SIZE_JSONPATH = "size"
     ATTR_HASH_JSONPATH = "checksum"
     ATTR_HASH_TYPE_VALUE = "sha256"
+
+
+class DataEuropaDataset(DatasetDownloader):
+    """Downloader for European data repository."""
+
+    REGEXP_ID = r"data\.europa\.eu\/data\/datasets\/(?P<record_id>.+)"
+
+    # the base entry point of the REST API
+    API_URL = "https://data.europa.eu/api/hub/repo/"
+
+    API_URL_META = "{api_url}datasets/{record_id}"
+    META_FILES_JSONPATH = '$.@graph[?(@.@type == "dcat:Distribution")]'
+
+    # paths to file attributes
+    ATTR_FILE_LINK_JSONPATH = "'dcat:accessURL'.@id"
+    ATTR_NAME_JSONPATH = "'dct:title'"
+    ATTR_SIZE_JSONPATH = "'dcat:byteSize'.@value"

tests/test_repositories.toml

@@ -113,3 +113,7 @@ files = "cbsodata-main/README.md"
 [[seanoe]]
 location = "https://doi.org/10.17882/101042"
 files = "111609.xlsx"
+
+[[dataeuropa]]
+location = "https://data.europa.eu/data/datasets/65e092e4009f18f050b14216"
+files = "consolidation-wattzhub-schema-irve-dynamic-20240918-033000.csv"