Merge branch 'main' into refac/cloud

.github/scripts/upload_test_stats_datadog.py

@@ -21,10 +21,24 @@
     "tests/e2e/test_notebooks.py::test_all_recipes[tests/e2e/import_predictions.ipynb]": (
         "import_predictions_ipynb"
     ),
+    "tests/e2e/test_notebooks.py::test_all_recipes[recipes/export_a_kili_project.ipynb]": (
+        "export_a_kili_project_ipynb"
+    ),
+    "tests/e2e/test_notebooks.py::test_all_recipes[recipes/importing_video_assets.ipynb]": (
+        "importing_video_assets_ipynb"
+    ),
     "tests/e2e/test_copy_project.py::test_copy_project_e2e_video": "copy_project_e2e_video",
     "tests/e2e/test_copy_project.py::test_copy_project_e2e_with_ocr_metadata": (
         "copy_project_e2e_with_ocr_metadata"
     ),
+    "tests/e2e/test_projects.py::test_create_project": "create_project",
+    "tests/e2e/test_mutations_label.py::test_append_many_labels": "append_many_labels",
+    "tests/e2e/test_mutations_asset.py::test_append_many_assets": "append_many_assets",
+    "tests/e2e/test_mutations_asset.py::test_send_back_to_queue": "send_back_to_queue",
+    "tests/e2e/test_mutations_asset.py::test_delete_many_from_dataset": "delete_many_from_dataset",
+    "tests/e2e/test_mutations_asset.py::test_change_asset_external_ids": (
+        "change_asset_external_ids"
+    ),
 }
 
 OWNER = "kili-technology"

.github/workflows/ci.yml

@@ -139,10 +139,11 @@ jobs:
 
       - name: Build
         run: |
-          mkdocs build 2>&1 | tee doc-build.log
+          mkdocs build
 
       - name: Look for Warnings
         run: |
+          mkdocs build 2>&1 | tee doc-build.log
           if grep -q "WARNING" doc-build.log; then
             echo $(grep "WARNING" doc-build.log)
             echo "::error::Documentation build completed with warnings"

docs/sdk/issue.md

@@ -1,5 +1,3 @@
 # Issue module
 
 ::: kili.presentation.client.issue.IssueClientMethods
-::: kili.entrypoints.queries.issue.__init__.QueriesIssue
-::: kili.entrypoints.mutations.issue.__init__.MutationsIssue

docs/sdk/project.md

@@ -2,6 +2,4 @@
 
 ::: kili.presentation.client.project.ProjectClientMethods
 
-::: kili.entrypoints.queries.project.__init__.QueriesProject
-
 ::: kili.entrypoints.mutations.project.__init__.MutationsProject

pyproject.toml

@@ -77,7 +77,7 @@ dev = [
   # linting
   "black",
   "pre-commit >= 3.3.0, < 4.0.0",
-  "pylint ==3.0.0",
+  "pylint ==3.0.1",
   "pyright == 1.1.329",
   # notebooks tests
   "nbformat",

src/kili/entrypoints/mutations/asset/__init__.py

@@ -73,8 +73,7 @@ def append_many_to_dataset(
                 If None, random identifiers are created.
             id_array: Disabled parameter. Do not use.
             is_honeypot_array:  Whether to use the asset for honeypot
-            status_array: By default, all imported assets are set to `TODO`. Other options:
-                `ONGOING`, `LABELED`, `REVIEWED`.
+            status_array: DEPRECATED and does not have any effect.
             json_content_array: Useful for `VIDEO` or `TEXT` projects only.
 
                 - For `VIDEO` projects, each element is a sequence of frames, i.e. a
@@ -88,7 +87,9 @@ def append_many_to_dataset(
                     Example for one asset: `json_metadata_array = [{'imageUrl': '','text': '','url': ''}]`.
                 - For VIDEO projects (and not VIDEO_LEGACY), you can specify a value with key 'processingParameters' to specify the sampling rate (default: 30).
                     Example for one asset: `json_metadata_array = [{'processingParameters': {'framesPlayedPerSecond': 10}}]`.
-                - For Image projects, if you work with geotiff, you can specify a value with key 'processingParameters' to specify the minimum and maximum zoom level.
+                - In Image projects with geoTIFF assets, you can specify the `minZoom` and `maxZoom` values for the `processingParameters` key.
+                    The `minZoom` parameter defines the zoom level that users are not allowed to zoom out from.
+                    The `maxZoom` value affects asset generation: the higher the value, the greater the level of details and the size of the asset.
                     Example for one asset: `json_metadata_array = [{'processingParameters': {'minZoom': 17, 'maxZoom': 19}}]`.
             disable_tqdm: If `True`, the progress bar will be disabled
             wait_until_availability: If `True`, the function will return once the assets are fully imported in Kili.
@@ -134,8 +135,8 @@ def append_many_to_dataset(
 
         if status_array is not None:
             warnings.warn(
-                "status_array is deprecated, asset status is automatically computed based on"
-                " its labels and cannot be overwritten.",
+                "status_array is deprecated and will not be sent in the call. Asset status is"
+                " automatically computed based on its labels and cannot be overwritten.",
                 DeprecationWarning,
                 stacklevel=1,
             )
@@ -154,7 +155,6 @@ def append_many_to_dataset(
             "json_content": json_content_array,
             "external_id": external_id_array,
             "id": id_array,
-            "status": status_array,
             "json_metadata": json_metadata_array,
             "is_honeypot": is_honeypot_array,
         }
@@ -213,8 +213,7 @@ def update_properties_in_assets(
                 is a text formatted using RichText.
                 - For a Video project, the`json_content` is a json containg urls pointing
                     to each frame of the video.
-            status_array: Each element should be in `TODO`, `ONGOING`, `LABELED`,
-                `TO_REVIEW`, `REVIEWED`.
+            status_array: DEPRECATED and does not have any effect.
             is_used_for_consensus_array: Whether to use the asset to compute consensus kpis or not.
             is_honeypot_array: Whether to use the asset for honeypot.
             project_id: The project ID. Only required if `external_ids` argument is provided.
@@ -238,7 +237,6 @@ def update_properties_in_assets(
                     is_honeypot_array=[True, True],
                     is_used_for_consensus_array=[True, False],
                     priorities=[None, 2],
-                    status_array=['LABELED', 'REVIEWED'],
                     to_be_labeled_by_array=[['[email protected]'], None],
                 )
 
@@ -266,12 +264,11 @@ def update_properties_in_assets(
 
         if status_array is not None:
             warnings.warn(
-                "status_array is deprecated, asset status is automatically computed based on"
-                " its labels and cannot be overwritten.",
+                "status_array is deprecated and will not be sent in the call. Asset status is"
+                " automatically computed based on its labels and cannot be overwritten.",
                 DeprecationWarning,
                 stacklevel=1,
             )
-
         if asset_ids is not None and external_ids is not None:
             warnings.warn(
                 "The use of `external_ids` argument has changed. It is now used to identify"
@@ -294,7 +291,6 @@ def update_properties_in_assets(
             to_be_labeled_by_array=to_be_labeled_by_array,
             contents=contents,
             json_contents=json_contents,
-            status_array=status_array,
             is_used_for_consensus_array=is_used_for_consensus_array,
             is_honeypot_array=is_honeypot_array,
             resolution_array=resolution_array,

src/kili/entrypoints/mutations/asset/helpers.py

@@ -17,7 +17,6 @@ def process_update_properties_in_assets_parameters(
     to_be_labeled_by_array: Optional[List[List[str]]] = None,
     contents: Optional[List[str]] = None,
     json_contents: Optional[List[str]] = None,
-    status_array: Optional[List[str]] = None,
     is_used_for_consensus_array: Optional[List[bool]] = None,
     is_honeypot_array: Optional[List[bool]] = None,
     resolution_array: Optional[List[dict]] = None,
@@ -47,7 +46,6 @@ def process_update_properties_in_assets_parameters(
         "shouldResetToBeLabeledBy": to_be_labeled_by_array,
         "content": contents,
         "jsonContent": json_contents,
-        "status": status_array,
         "isUsedForConsensus": is_used_for_consensus_array,
         "isHoneypot": is_honeypot_array,
         "pageResolutions": page_resolutions_array,

src/kili/services/copy_project/__init__.py

@@ -26,7 +26,7 @@ class ProjectCopier:  # pylint: disable=too-few-public-methods
         "inputType",
         "description",
         "id",
-        "dataConnections.dataIntegrationId",
+        "dataConnections.id",
     )
     FIELDS_JSON_INTERFACE = ("jsonInterface",)
     FIELDS_QUALITY_SETTINGS = (
@@ -87,7 +87,7 @@ def copy_project(  # pylint: disable=too-many-arguments,too-many-locals
 
         src_project = get_project(self.kili, from_project_id, fields)
 
-        if len(src_project["dataConnections"]) > 0 and copy_assets:
+        if src_project["dataConnections"] and copy_assets:
             raise NotImplementedError("Copying projects with cloud storage is not supported.")
 
         new_project_title = title or self._generate_project_title(src_title=src_project["title"])

src/kili/services/export/format/base.py

@@ -10,8 +10,6 @@
 from pathlib import Path
 from typing import Dict, List, NamedTuple, Optional, Tuple, cast
 
-from kili.adapters.kili_api_gateway import KiliAPIGateway
-from kili.adapters.kili_api_gateway.helpers.queries import QueryOptions
 from kili.domain.asset import AssetId
 from kili.domain.cloud_storage import DataConnectionFilters
 from kili.domain.project import ProjectId
@@ -204,16 +202,8 @@ def _check_and_ensure_asset_access(self) -> None:
             )
 
     def _has_data_connection(self) -> bool:
-        data_connections_gen = KiliAPIGateway(
-            self.kili.graphql_client, self.kili.http_client
-        ).list_data_connections(
-            data_connection_filters=DataConnectionFilters(
-                project_id=self.project_id, integration_id=None
-            ),
-            options=QueryOptions(disable_tqdm=True, first=1, skip=0),
-            fields=("id",),
-        )
-        return len(list(data_connections_gen)) > 0
+        project = get_project(self.kili, self.project_id, ["dataConnections.id"])
+        return bool(project["dataConnections"])
 
     def _check_geotiff_export_compatibility(self, assets: List[Asset]) -> None:
         # pylint: disable=line-too-long

src/kili/use_cases/asset/media_downloader.py

@@ -13,8 +13,6 @@
 
 from kili.adapters.http_client import HttpClient
 from kili.adapters.kili_api_gateway import KiliAPIGateway
-from kili.adapters.kili_api_gateway.helpers.queries import QueryOptions
-from kili.domain.cloud_storage import DataConnectionFilters
 from kili.domain.project import ProjectId
 from kili.domain.types import ListOrTuple
 
@@ -38,17 +36,14 @@ def get_download_assets_function(
     if not download_media:
         return None, fields
 
-    project = kili_api_gateway.get_project(project_id=project_id, fields=("inputType",))
+    project = kili_api_gateway.get_project(
+        project_id=project_id, fields=("inputType", "dataConnections.id")
+    )
     input_type = project["inputType"]
 
     # We need to query the data connections to know if the assets are hosted in a cloud storage
     # If so, we remove the fields "content" and "jsonContent" from the query
-    data_connections_gen = kili_api_gateway.list_data_connections(
-        DataConnectionFilters(project_id=project_id, integration_id=None),
-        fields=("id",),
-        options=QueryOptions(disable_tqdm=True, first=1, skip=0),
-    )
-    if len(list(data_connections_gen)) > 0:
+    if project["dataConnections"]:
         raise DownloadNotAllowedError(
             "The download of assets from a project connected to a cloud storage is not allowed."
             " Asset download is disabled."

tests/e2e/test_mutations_asset.py

@@ -146,3 +146,49 @@ def test_update_properties_in_assets_external_id(kili, src_project):
 
     assert [asset["externalId"] for asset in assets_new] == ["1", "2", "3"]
     assert [asset["priority"] for asset in assets_new] == [1, 0, 0]
+
+
+@pytest.fixture()
+def src_project_no_assets(kili: Kili):
+    interface = {
+        "jobs": {
+            "DETECTION": {
+                "mlTask": "OBJECT_DETECTION",
+                "tools": ["rectangle"],
+                "instruction": "Is there a defect ? Where ? What kind ?",
+                "required": 0,
+                "isChild": False,
+                "content": {
+                    "categories": {
+                        "DEFECT_CLASS_1": {"name": "defect of class 1"},
+                        "DEFECT_CLASS_2": {"name": "defect of class 2"},
+                        "DEFECT_CLASS_3": {"name": "defect of class 3"},
+                        "DEFECT_CLASS_4": {"name": "defect of class 4"},
+                    },
+                    "input": "radio",
+                },
+            }
+        }
+    }
+
+    project = kili.create_project(
+        input_type="IMAGE",
+        json_interface=interface,
+        title="e2e test_add_many_assets",
+    )
+
+    yield project["id"]
+
+    kili.delete_project(project["id"])
+
+
+def test_append_many_assets(kili: Kili, src_project_no_assets: str):
+    NB_ASSETS = 500
+    img_url = "https://storage.googleapis.com/label-public-staging/car/car_1.jpg"
+
+    kili.append_many_to_dataset(
+        project_id=src_project_no_assets,
+        content_array=[img_url] * NB_ASSETS,
+    )
+
+    assert kili.count_assets(src_project_no_assets) == NB_ASSETS

tests/e2e/test_mutations_label.py

@@ -92,3 +92,51 @@ def test_e2e_append_labels_overwrite(kili: Kili, project: Dict):
     ]
     for label_id in old_pred_ids_asset_1:
         assert label_id not in new_label_ids
+
+
+@pytest.fixture()
+def project_and_asset_id(kili: Kili):
+    interface = {
+        "jobs": {
+            "CLASSIFICATION_JOB": {
+                "content": {"categories": {"A": {"children": [], "name": "A"}}, "input": "radio"},
+                "instruction": "classif",
+                "mlTask": "CLASSIFICATION",
+                "required": 1,
+                "isChild": False,
+            }
+        }
+    }
+
+    project = kili.create_project(
+        input_type="TEXT",
+        json_interface=interface,
+        title="test_append_many_labels",
+    )
+
+    kili.append_many_to_dataset(
+        project_id=project["id"],
+        content_array=["asset_content_1"],
+        external_id_array=["1"],
+    )
+
+    asset_id = kili.assets(project_id=project["id"], fields=("id",))[0]
+
+    yield project["id"], asset_id["id"]
+
+    kili.delete_project(project["id"])
+
+
+def test_append_many_labels(kili: Kili, project_and_asset_id):
+    project_id, asset_id = project_and_asset_id
+
+    N_LABELS = 1000
+
+    kili.append_labels(
+        project_id=project_id,
+        asset_id_array=[asset_id] * N_LABELS,
+        json_response_array=[{"CLASSIFICATION_JOB": {"categories": [{"name": "A"}]}}] * N_LABELS,
+        label_type="DEFAULT",
+    )
+
+    assert kili.count_labels(project_id=project_id) == N_LABELS

tests/e2e/test_projects.py

@@ -5,6 +5,20 @@
 from kili.client import Kili
 
 
+@pytest.fixture()
+def project_name(kili: Kili):
+    project_name = "e2e test_create_project " + str(uuid.uuid4())
+
+    yield project_name
+
+    project_id = kili.projects(search_query=f"%{project_name}%", first=1)[0]["id"]
+    kili.delete_project(project_id)
+
+
+def test_create_project(project_name: str, kili: Kili):
+    _ = kili.create_project(input_type="TEXT", json_interface={}, title=project_name)
+
+
 @pytest.fixture()
 def projects_uuid(kili: Kili):
     projects_uuid = str(uuid.uuid4())

tests/fakes/fake_kili.py

@@ -93,6 +93,7 @@ def mocked_ProjectQuery(where, _fields, _options):
                 "description": "This is a test project",
                 "jsonInterface": json_interface,
                 "inputType": "IMAGE",
+                "dataConnections": None,
             }
         ]
     elif project_id == "object_detection_video_project":
@@ -122,6 +123,7 @@ def mocked_ProjectQuery(where, _fields, _options):
                 "description": "This is a test project",
                 "jsonInterface": json_interface,
                 "inputType": "VIDEO",
+                "dataConnections": None,
             }
         ]
     elif project_id == "text_classification":
@@ -155,6 +157,7 @@ def mocked_ProjectQuery(where, _fields, _options):
                 "description": "This is a TC test project",
                 "jsonInterface": json_interface,
                 "inputType": "TEXT",
+                "dataConnections": None,
             }
         ]
     elif project_id == "semantic_segmentation":
@@ -198,6 +201,7 @@ def mocked_ProjectQuery(where, _fields, _options):
                 "description": "This is a semantic segmentation test project",
                 "jsonInterface": json_interface,
                 "inputType": "IMAGE",
+                "dataConnections": None,
             }
         ]
     else:

tests/integration/use_cases/test_asset.py

@@ -110,7 +110,7 @@ def test_given_query_parameters_I_can_query_assets_and_download_their_media(
     kili_api_gateway: KiliAPIGateway, mocker
 ):
     # mocking
-    kili_api_gateway.get_project.return_value = {"inputType": "IMAGE"}
+    kili_api_gateway.get_project.return_value = {"inputType": "IMAGE", "dataConnections": None}
     media_downlaoder_mock = mocker.patch.object(MediaDownloader, "__init__", return_value=None)
 
     # given parameters to query assets