fix(LAB-2577): AAU I want to be able to put no content when json_cont… (

#1645)

Co-authored-by: Aurelien Lombard <[email protected]>

.github/scripts/upload_test_stats_datadog.py

@@ -34,6 +34,7 @@
     "tests/e2e/test_projects.py::test_create_project": "create_project",
     "tests/e2e/test_mutations_label.py::test_append_many_labels": "append_many_labels",
     "tests/e2e/test_mutations_asset.py::test_append_many_assets": "append_many_assets",
+    "tests/e2e/test_mutations_asset.py::test_append_many_assets_with_json_content_but_no_content": "append_many_assets_with_json_content_but_no_content",
     "tests/e2e/test_mutations_asset.py::test_send_back_to_queue": "send_back_to_queue",
     "tests/e2e/test_mutations_asset.py::test_delete_many_from_dataset": "delete_many_from_dataset",
     "tests/e2e/test_mutations_asset.py::test_change_asset_external_ids": (

src/kili/entrypoints/mutations/asset/__init__.py

@@ -48,7 +48,7 @@ def append_many_to_dataset(
         id_array: Optional[List[str]] = None,
         is_honeypot_array: Optional[List[bool]] = None,
         status_array: Optional[List[str]] = None,
-        json_content_array: Optional[List[List[Union[dict, str]]]] = None,
+        json_content_array: Optional[List[Union[List[Union[dict, str]], None]]] = None,
         json_metadata_array: Optional[List[dict]] = None,
         disable_tqdm: Optional[bool] = None,
         wait_until_availability: bool = True,
@@ -75,13 +75,17 @@ def append_many_to_dataset(
             id_array: Disabled parameter. Do not use.
             is_honeypot_array:  Whether to use the asset for honeypot
             status_array: DEPRECATED and does not have any effect.
-            json_content_array: Useful for `VIDEO` or `TEXT` projects only.
+            json_content_array: Useful for `VIDEO` or `TEXT` or `IMAGE` projects only.
 
                 - For `VIDEO` projects, each element is a sequence of frames, i.e. a
                     list of URLs to images or a list of paths to images.
                 - For `TEXT` projects, each element is a json_content dict,
                     formatted according to documentation [on how to import
                 rich-text assets](https://python-sdk-docs.kili-technology.com/latest/sdk/tutorials/import_text_assets/).
+                - For `IMAGES` projects, it is used for satellite imagery each element is a list of json_content dicts
+                    formatted according to documentation [on how to add multi-layer images]
+                    (https://docs.kili-technology.com/docs/adding-assets-to-project#adding-multi-layer-images)
+
             json_metadata_array: The metadata given to each asset should be stored in a json like dict with keys.
 
                 - Add metadata visible on the asset with the following keys: `imageUrl`, `text`, `url`.

src/kili/services/asset_import/__init__.py

@@ -47,4 +47,6 @@ def import_assets(  # pylint: disable=too-many-arguments
     if input_type not in importer_by_type:
         raise NotImplementedError(f"There is no imported for the input type: {input_type}")
     asset_importer = importer_by_type[input_type](*importer_params)
-    return asset_importer.import_assets(assets=cast(List[AssetLike], assets))
+    casted_assets = cast(List[AssetLike], assets)
+    asset_importer.check_asset_contents(casted_assets)
+    return asset_importer.import_assets(assets=casted_assets)

src/kili/services/asset_import/base.py

@@ -257,7 +257,10 @@ def import_batch(self, assets: List[AssetLike], verify: bool):
         """Method to import a batch of asset with content."""
         assets = self.add_ids(assets)
         if not self.is_hosted:
-            assets = self.upload_local_content_to_bucket(assets)
+            assets_with_content = [asset for asset in assets if asset.get("content")]
+            assets = [asset for asset in assets if not asset.get("content")]
+            if len(assets_with_content) > 0:
+                assets += self.upload_local_content_to_bucket(assets_with_content)
         return super().import_batch(assets, verify)
 
     def get_content_type_and_data_from_content(
@@ -378,7 +381,7 @@ def is_hosted_content(assets: List[AssetLike]) -> bool:
 
         Raise an error if a mix of both.
         """
-        contents = [asset.get("content") for asset in assets]
+        contents = [asset.get("content") for asset in assets if asset.get("content")]
         if all(is_url(content) for content in contents):
             return True
         if any(is_url(content) for content in contents):
@@ -388,6 +391,19 @@ def is_hosted_content(assets: List[AssetLike]) -> bool:
             )
         return False
 
+    @staticmethod
+    def check_asset_contents(assets: List[AssetLike]) -> None:
+        """Determine if the assets have at least one content or json_content.
+
+        Raise an error if not
+        """
+        # Raise an error if there is an asset with no content and no json_content
+        for asset in assets:
+            if not asset.get("content") and not asset.get("json_content"):
+                raise ImportValidationError(
+                    "Cannot import asset with empty content and empty json_content"
+                )
+
     def _can_upload_from_local_data(self) -> bool:
         user_me = self.kili.kili_api_gateway.get_current_user(fields=("email",))
         options = QueryOptions(first=1, disable_tqdm=True)
@@ -415,7 +431,11 @@ def filter_local_assets(self, assets: List[AssetLike], raise_error: bool):
         """
         filtered_assets = []
         for asset in assets:
+            json_content = asset.get("json_content")
             path = asset.get("content")
+            if json_content and not path:
+                filtered_assets.append(asset)
+                continue
             assert path
             assert isinstance(path, str)
             try:

src/kili/services/asset_import/image.py

@@ -44,7 +44,11 @@ def split_asset_by_upload_type(assets: List[AssetLike], is_hosted: bool):
             return assets, []
         sync_assets, async_assets = [], []
         for asset in assets:
+            json_content = asset.get("json_content")
             path = asset.get("content")
+            if json_content and not path:
+                sync_assets.append(asset)
+                continue
             assert path
             assert isinstance(path, str)
             mime_type = get_mime_type(path)

tests/e2e/test_mutations_asset.py

@@ -1,3 +1,5 @@
+from typing import List, Union
+
 import pytest
 
 from kili.client import Kili
@@ -192,3 +194,30 @@ def test_append_many_assets(kili: Kili, src_project_no_assets: str):
     )
 
     assert kili.count_assets(src_project_no_assets) == NB_ASSETS
+
+
+def test_append_many_assets_with_json_content_but_no_content(
+    kili: Kili, src_project_no_assets: str
+):
+    json_content: List[Union[dict, str]] = [
+        {
+            "bounds": [[100.78454549662813, 13.660975292601123], [100.721440891, 13.719807693]],
+            "epsg": "EPSG4326",
+            "tileLayerUrl": "http://localhost:5005/map1/{z}/{x}/{y}.png",
+            "effective_zoom_levels": [14, 15, 16, 17],
+            "minZoom": 14,
+            "maxZoom": 17,
+            "initEpsg": 4326,
+            "useClassicCoordinates": False,
+        }
+    ]
+    img_url = "https://storage.googleapis.com/label-public-staging/car/car_1.jpg"
+
+    kili.append_many_to_dataset(
+        project_id=src_project_no_assets,
+        content_array=["", img_url],
+        json_content_array=[json_content, None],
+        external_id_array=["asset_1", "asset_2"],
+    )
+
+    assert kili.count_assets(src_project_no_assets) == 2