From baa686925597ff1e24f0156e44a9b312b6060590 Mon Sep 17 00:00:00 2001 From: aurelienlombard <123479007+aurelienlombard@users.noreply.github.com> Date: Mon, 5 Feb 2024 11:20:01 +0100 Subject: [PATCH] =?UTF-8?q?fix(LAB-2577):=20AAU=20I=20want=20to=20be=20abl?= =?UTF-8?q?e=20to=20put=20no=20content=20when=20json=5Fcont=E2=80=A6=20(#1?= =?UTF-8?q?645)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Aurelien Lombard --- .github/scripts/upload_test_stats_datadog.py | 1 + .../entrypoints/mutations/asset/__init__.py | 8 +++-- src/kili/services/asset_import/__init__.py | 4 ++- src/kili/services/asset_import/base.py | 24 +++++++++++++-- src/kili/services/asset_import/image.py | 4 +++ tests/e2e/test_mutations_asset.py | 29 +++++++++++++++++++ 6 files changed, 65 insertions(+), 5 deletions(-) diff --git a/.github/scripts/upload_test_stats_datadog.py b/.github/scripts/upload_test_stats_datadog.py index 154c8e1ed..9a9cbe624 100644 --- a/.github/scripts/upload_test_stats_datadog.py +++ b/.github/scripts/upload_test_stats_datadog.py @@ -34,6 +34,7 @@ "tests/e2e/test_projects.py::test_create_project": "create_project", "tests/e2e/test_mutations_label.py::test_append_many_labels": "append_many_labels", "tests/e2e/test_mutations_asset.py::test_append_many_assets": "append_many_assets", + "tests/e2e/test_mutations_asset.py::test_append_many_assets_with_json_content_but_no_content": "append_many_assets_with_json_content_but_no_content", "tests/e2e/test_mutations_asset.py::test_send_back_to_queue": "send_back_to_queue", "tests/e2e/test_mutations_asset.py::test_delete_many_from_dataset": "delete_many_from_dataset", "tests/e2e/test_mutations_asset.py::test_change_asset_external_ids": ( diff --git a/src/kili/entrypoints/mutations/asset/__init__.py b/src/kili/entrypoints/mutations/asset/__init__.py index dcd457818..b230af65d 100644 --- a/src/kili/entrypoints/mutations/asset/__init__.py +++ b/src/kili/entrypoints/mutations/asset/__init__.py @@ -48,7 +48,7 @@ def append_many_to_dataset( id_array: Optional[List[str]] = None, is_honeypot_array: Optional[List[bool]] = None, status_array: Optional[List[str]] = None, - json_content_array: Optional[List[List[Union[dict, str]]]] = None, + json_content_array: Optional[List[Union[List[Union[dict, str]], None]]] = None, json_metadata_array: Optional[List[dict]] = None, disable_tqdm: Optional[bool] = None, wait_until_availability: bool = True, @@ -75,13 +75,17 @@ def append_many_to_dataset( id_array: Disabled parameter. Do not use. is_honeypot_array: Whether to use the asset for honeypot status_array: DEPRECATED and does not have any effect. - json_content_array: Useful for `VIDEO` or `TEXT` projects only. + json_content_array: Useful for `VIDEO` or `TEXT` or `IMAGE` projects only. - For `VIDEO` projects, each element is a sequence of frames, i.e. a list of URLs to images or a list of paths to images. - For `TEXT` projects, each element is a json_content dict, formatted according to documentation [on how to import rich-text assets](https://python-sdk-docs.kili-technology.com/latest/sdk/tutorials/import_text_assets/). + - For `IMAGES` projects, it is used for satellite imagery each element is a list of json_content dicts + formatted according to documentation [on how to add multi-layer images] + (https://docs.kili-technology.com/docs/adding-assets-to-project#adding-multi-layer-images) + json_metadata_array: The metadata given to each asset should be stored in a json like dict with keys. - Add metadata visible on the asset with the following keys: `imageUrl`, `text`, `url`. diff --git a/src/kili/services/asset_import/__init__.py b/src/kili/services/asset_import/__init__.py index 5682adb66..e78b2356d 100644 --- a/src/kili/services/asset_import/__init__.py +++ b/src/kili/services/asset_import/__init__.py @@ -47,4 +47,6 @@ def import_assets( # pylint: disable=too-many-arguments if input_type not in importer_by_type: raise NotImplementedError(f"There is no imported for the input type: {input_type}") asset_importer = importer_by_type[input_type](*importer_params) - return asset_importer.import_assets(assets=cast(List[AssetLike], assets)) + casted_assets = cast(List[AssetLike], assets) + asset_importer.check_asset_contents(casted_assets) + return asset_importer.import_assets(assets=casted_assets) diff --git a/src/kili/services/asset_import/base.py b/src/kili/services/asset_import/base.py index 1d654f5e5..04337fc2a 100644 --- a/src/kili/services/asset_import/base.py +++ b/src/kili/services/asset_import/base.py @@ -257,7 +257,10 @@ def import_batch(self, assets: List[AssetLike], verify: bool): """Method to import a batch of asset with content.""" assets = self.add_ids(assets) if not self.is_hosted: - assets = self.upload_local_content_to_bucket(assets) + assets_with_content = [asset for asset in assets if asset.get("content")] + assets = [asset for asset in assets if not asset.get("content")] + if len(assets_with_content) > 0: + assets += self.upload_local_content_to_bucket(assets_with_content) return super().import_batch(assets, verify) def get_content_type_and_data_from_content( @@ -378,7 +381,7 @@ def is_hosted_content(assets: List[AssetLike]) -> bool: Raise an error if a mix of both. """ - contents = [asset.get("content") for asset in assets] + contents = [asset.get("content") for asset in assets if asset.get("content")] if all(is_url(content) for content in contents): return True if any(is_url(content) for content in contents): @@ -388,6 +391,19 @@ def is_hosted_content(assets: List[AssetLike]) -> bool: ) return False + @staticmethod + def check_asset_contents(assets: List[AssetLike]) -> None: + """Determine if the assets have at least one content or json_content. + + Raise an error if not + """ + # Raise an error if there is an asset with no content and no json_content + for asset in assets: + if not asset.get("content") and not asset.get("json_content"): + raise ImportValidationError( + "Cannot import asset with empty content and empty json_content" + ) + def _can_upload_from_local_data(self) -> bool: user_me = self.kili.kili_api_gateway.get_current_user(fields=("email",)) options = QueryOptions(first=1, disable_tqdm=True) @@ -415,7 +431,11 @@ def filter_local_assets(self, assets: List[AssetLike], raise_error: bool): """ filtered_assets = [] for asset in assets: + json_content = asset.get("json_content") path = asset.get("content") + if json_content and not path: + filtered_assets.append(asset) + continue assert path assert isinstance(path, str) try: diff --git a/src/kili/services/asset_import/image.py b/src/kili/services/asset_import/image.py index b10079f7a..d0e32d43d 100644 --- a/src/kili/services/asset_import/image.py +++ b/src/kili/services/asset_import/image.py @@ -44,7 +44,11 @@ def split_asset_by_upload_type(assets: List[AssetLike], is_hosted: bool): return assets, [] sync_assets, async_assets = [], [] for asset in assets: + json_content = asset.get("json_content") path = asset.get("content") + if json_content and not path: + sync_assets.append(asset) + continue assert path assert isinstance(path, str) mime_type = get_mime_type(path) diff --git a/tests/e2e/test_mutations_asset.py b/tests/e2e/test_mutations_asset.py index 7d811a8d1..85f1a4fc7 100644 --- a/tests/e2e/test_mutations_asset.py +++ b/tests/e2e/test_mutations_asset.py @@ -1,3 +1,5 @@ +from typing import List, Union + import pytest from kili.client import Kili @@ -192,3 +194,30 @@ def test_append_many_assets(kili: Kili, src_project_no_assets: str): ) assert kili.count_assets(src_project_no_assets) == NB_ASSETS + + +def test_append_many_assets_with_json_content_but_no_content( + kili: Kili, src_project_no_assets: str +): + json_content: List[Union[dict, str]] = [ + { + "bounds": [[100.78454549662813, 13.660975292601123], [100.721440891, 13.719807693]], + "epsg": "EPSG4326", + "tileLayerUrl": "http://localhost:5005/map1/{z}/{x}/{y}.png", + "effective_zoom_levels": [14, 15, 16, 17], + "minZoom": 14, + "maxZoom": 17, + "initEpsg": 4326, + "useClassicCoordinates": False, + } + ] + img_url = "https://storage.googleapis.com/label-public-staging/car/car_1.jpg" + + kili.append_many_to_dataset( + project_id=src_project_no_assets, + content_array=["", img_url], + json_content_array=[json_content, None], + external_id_array=["asset_1", "asset_2"], + ) + + assert kili.count_assets(src_project_no_assets) == 2