diff --git a/src/kili/services/copy_project/__init__.py b/src/kili/services/copy_project/__init__.py index a89d69ae2..4a33f42e7 100644 --- a/src/kili/services/copy_project/__init__.py +++ b/src/kili/services/copy_project/__init__.py @@ -26,7 +26,7 @@ class ProjectCopier: # pylint: disable=too-few-public-methods "inputType", "description", "id", - "dataConnections.dataIntegrationId", + "dataConnections.id", ) FIELDS_JSON_INTERFACE = ("jsonInterface",) FIELDS_QUALITY_SETTINGS = ( @@ -87,7 +87,7 @@ def copy_project( # pylint: disable=too-many-arguments,too-many-locals src_project = get_project(self.kili, from_project_id, fields) - if len(src_project["dataConnections"]) > 0 and copy_assets: + if src_project["dataConnections"] and copy_assets: raise NotImplementedError("Copying projects with cloud storage is not supported.") new_project_title = title or self._generate_project_title(src_title=src_project["title"]) diff --git a/src/kili/services/export/format/base.py b/src/kili/services/export/format/base.py index 7f262e26f..e37aaf4ed 100644 --- a/src/kili/services/export/format/base.py +++ b/src/kili/services/export/format/base.py @@ -10,11 +10,6 @@ from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Tuple, cast -from kili.adapters.kili_api_gateway.helpers.queries import QueryOptions -from kili.core.graphql.operations.data_connection.queries import ( - DataConnectionsQuery, - DataConnectionsWhere, -) from kili.domain.asset import AssetId from kili.domain.project import ProjectId from kili.orm import Asset, Label @@ -206,14 +201,8 @@ def _check_and_ensure_asset_access(self) -> None: ) def _has_data_connection(self) -> bool: - data_connections_gen = DataConnectionsQuery( - self.kili.graphql_client, self.kili.http_client - )( - where=DataConnectionsWhere(project_id=self.project_id), - fields=["id"], - options=QueryOptions(disable_tqdm=True, first=1, skip=0), - ) - return len(list(data_connections_gen)) > 0 + project = get_project(self.kili, self.project_id, ["dataConnections.id"]) + return bool(project["dataConnections"]) def _check_geotiff_export_compatibility(self, assets: List[Asset]) -> None: # pylint: disable=line-too-long diff --git a/src/kili/use_cases/asset/media_downloader.py b/src/kili/use_cases/asset/media_downloader.py index aeaf2d06b..a5d277d6d 100644 --- a/src/kili/use_cases/asset/media_downloader.py +++ b/src/kili/use_cases/asset/media_downloader.py @@ -13,11 +13,6 @@ from kili.adapters.http_client import HttpClient from kili.adapters.kili_api_gateway import KiliAPIGateway -from kili.adapters.kili_api_gateway.helpers.queries import QueryOptions -from kili.core.graphql.operations.data_connection.queries import ( - DataConnectionsQuery, - DataConnectionsWhere, -) from kili.domain.project import ProjectId from kili.domain.types import ListOrTuple @@ -41,19 +36,14 @@ def get_download_assets_function( if not download_media: return None, fields - project = kili_api_gateway.get_project(project_id=project_id, fields=("inputType",)) + project = kili_api_gateway.get_project( + project_id=project_id, fields=("inputType", "dataConnections.id") + ) input_type = project["inputType"] # We need to query the data connections to know if the assets are hosted in a cloud storage # If so, we remove the fields "content" and "jsonContent" from the query - data_connections_gen = DataConnectionsQuery( - kili_api_gateway.graphql_client, kili_api_gateway.http_client - )( - where=DataConnectionsWhere(project_id=project_id), - fields=("id",), - options=QueryOptions(disable_tqdm=True, first=1, skip=0), - ) - if len(list(data_connections_gen)) > 0: + if project["dataConnections"]: raise DownloadNotAllowedError( "The download of assets from a project connected to a cloud storage is not allowed." " Asset download is disabled." diff --git a/tests/e2e/test_notebooks.py b/tests/e2e/test_notebooks.py index eb007e225..64664e780 100644 --- a/tests/e2e/test_notebooks.py +++ b/tests/e2e/test_notebooks.py @@ -25,7 +25,7 @@ def process_notebook(notebook_filename: str) -> None: pytest.param( "tests/e2e/plugin_workflow.ipynb", marks=pytest.mark.skipif( - "lts.cloud" in os.environ["KILI_API_ENDPOINT"], + "lts.cloud" in os.getenv("KILI_API_ENDPOINT", ""), reason="Feature not available on premise", ), ), @@ -50,7 +50,7 @@ def process_notebook(notebook_filename: str) -> None: pytest.param( "recipes/plugins_example.ipynb", marks=pytest.mark.skipif( - "lts.cloud" in os.environ["KILI_API_ENDPOINT"], + "lts.cloud" in os.getenv("KILI_API_ENDPOINT", ""), reason="Feature not available on premise", ), ), diff --git a/tests/fakes/fake_kili.py b/tests/fakes/fake_kili.py index 27b4d3785..76cf668f5 100644 --- a/tests/fakes/fake_kili.py +++ b/tests/fakes/fake_kili.py @@ -93,6 +93,7 @@ def mocked_ProjectQuery(where, _fields, _options): "description": "This is a test project", "jsonInterface": json_interface, "inputType": "IMAGE", + "dataConnections": None, } ] elif project_id == "object_detection_video_project": @@ -122,6 +123,7 @@ def mocked_ProjectQuery(where, _fields, _options): "description": "This is a test project", "jsonInterface": json_interface, "inputType": "VIDEO", + "dataConnections": None, } ] elif project_id == "text_classification": @@ -155,6 +157,7 @@ def mocked_ProjectQuery(where, _fields, _options): "description": "This is a TC test project", "jsonInterface": json_interface, "inputType": "TEXT", + "dataConnections": None, } ] elif project_id == "semantic_segmentation": @@ -198,6 +201,7 @@ def mocked_ProjectQuery(where, _fields, _options): "description": "This is a semantic segmentation test project", "jsonInterface": json_interface, "inputType": "IMAGE", + "dataConnections": None, } ] else: diff --git a/tests/integration/use_cases/test_asset.py b/tests/integration/use_cases/test_asset.py index 0007fc80c..73eb413f0 100644 --- a/tests/integration/use_cases/test_asset.py +++ b/tests/integration/use_cases/test_asset.py @@ -110,7 +110,7 @@ def test_given_query_parameters_I_can_query_assets_and_download_their_media( kili_api_gateway: KiliAPIGateway, mocker ): # mocking - kili_api_gateway.get_project.return_value = {"inputType": "IMAGE"} + kili_api_gateway.get_project.return_value = {"inputType": "IMAGE", "dataConnections": None} media_downlaoder_mock = mocker.patch.object(MediaDownloader, "__init__", return_value=None) # given parameters to query assets diff --git a/tests/unit/services/export/test_coco.py b/tests/unit/services/export/test_coco.py index d9d44a25f..b173fec9f 100644 --- a/tests/unit/services/export/test_coco.py +++ b/tests/unit/services/export/test_coco.py @@ -16,7 +16,6 @@ _get_coco_categories_with_mapping, _get_coco_geometry_from_kili_bpoly, ) -from kili.services.export.format.kili import KiliExporter from kili.services.types import Job, JobName from kili.utils.tempfile import TemporaryDirectory @@ -579,12 +578,9 @@ def test_when_exporting_to_coco_given_a_project_with_data_connection_then_it_sho mocker.patch( "kili.services.export.format.base.get_project", return_value=get_project_return_val ) - mocker.patch.object(KiliExporter, "_check_arguments_compatibility", return_value=None) - mocker.patch.object(KiliExporter, "_check_project_compatibility", return_value=None) - mocker.patch( - "kili.services.export.format.base.DataConnectionsQuery.__call__", - return_value=(i for i in [{"id": "fake_data_connection_id"}]), - ) + mocker.patch.object(CocoExporter, "_check_arguments_compatibility", return_value=None) + mocker.patch.object(CocoExporter, "_check_project_compatibility", return_value=None) + mocker.patch.object(CocoExporter, "_has_data_connection", return_value=True) kili = QueriesLabel() kili.api_key = "" # type: ignore diff --git a/tests/unit/services/export/test_export.py b/tests/unit/services/export/test_export.py index be5f07e08..6941b04f6 100644 --- a/tests/unit/services/export/test_export.py +++ b/tests/unit/services/export/test_export.py @@ -14,13 +14,14 @@ from kili.domain.asset import AssetFilters from kili.entrypoints.queries.label import QueriesLabel from kili.orm import Asset -from kili.services.export import export_labels +from kili.services.export import AbstractExporter, export_labels from kili.services.export.exceptions import ( NoCompatibleJobError, NotCompatibleInputType, NotCompatibleOptions, ) from kili.services.export.format.kili import KiliExporter +from kili.services.export.format.voc import VocExporter from tests.fakes.fake_kili import ( FakeKili, mocked_AssetQuery, @@ -794,6 +795,7 @@ def test_export_with_asset_filter_kwargs(mocker): "kili.services.export.format.base.get_project", return_value=get_project_return_val ) mocker.patch.object(KiliExporter, "process_and_save", return_value=None) + mocker.patch.object(KiliExporter, "_has_data_connection", return_value=False) kili = QueriesLabel() kili.api_endpoint = "https://" # type: ignore kili.api_key = "" # type: ignore @@ -875,6 +877,7 @@ def test_export_with_asset_filter_kwargs_unknown_arg(mocker): ) mocker.patch.object(KiliExporter, "_check_arguments_compatibility", return_value=None) mocker.patch.object(KiliExporter, "_check_project_compatibility", return_value=None) + mocker.patch.object(KiliExporter, "_has_data_connection", return_value=False) kili = QueriesLabel() kili.api_endpoint = "https://" # type: ignore kili.api_key = "" # type: ignore @@ -917,11 +920,7 @@ def mock_kili(mocker, with_data_connection): mocker.patch( "kili.services.export.format.base.get_project", return_value=get_project_return_val ) - if with_data_connection: - mocker.patch( - "kili.services.export.format.base.DataConnectionsQuery.__call__", - return_value=(i for i in [{"id": "fake_data_connection_id"}]), - ) + mocker.patch.object(AbstractExporter, "_has_data_connection", return_value=with_data_connection) kili = QueriesLabel() kili.kili_api_gateway = mocker.MagicMock() @@ -1050,3 +1049,59 @@ def test_when_exporting_geotiff_asset_with_incompatible_options_then_it_crashes( ), ): kili.export_labels("fake_proj_id", "export.zip", fmt="kili", normalized_coordinates=False) + + +def test_given_kili_when_exporting_it_does_not_call_dataconnection_resolver( + mocker: pytest_mock.MockerFixture, +): + """Test that the dataconnection resolver is not called when exporting. + + Export for projects with data connections is forbidden. + But dataConnections() resolver requires high permissions. + This test ensures that the resolver is not called when exporting. + """ + # Given + project_return_val = { + "jsonInterface": { + "jobs": { + "OBJECT_DETECTION_JOB": { + "content": { + "categories": { + "GDGF": { + "children": [], + "color": "#472CED", + "name": "gdgf", + } + }, + "input": "radio", + }, + "instruction": "df", + "mlTask": "OBJECT_DETECTION", + "required": 1, + "tools": ["rectangle"], + "isChild": False, + } + } + }, + "inputType": "IMAGE", + "title": "", + "dataConnections": None, + } + mocker.patch.object(ProjectQuery, "__call__", return_value=[project_return_val]) + mocker.patch("kili.services.export.format.base.fetch_assets", return_value=[]) + process_and_save_mock = mocker.patch.object(VocExporter, "process_and_save", return_value=None) + kili = QueriesLabel() + kili.api_endpoint = "https://" # type: ignore + kili.api_key = "" # type: ignore + kili.graphql_client = mocker.MagicMock() + kili.http_client = mocker.MagicMock() + kili.kili_api_gateway = mocker.MagicMock() + + # When + kili.export_labels( + project_id="fake_proj_id", filename="exp.zip", fmt="pascal_voc", layout="merged" + ) + + # Then + process_and_save_mock.assert_called_once() + kili.graphql_client.execute.assert_not_called() diff --git a/tests/unit/services/export/test_geojson.py b/tests/unit/services/export/test_geojson.py index 51ab7036b..5c13da0f3 100644 --- a/tests/unit/services/export/test_geojson.py +++ b/tests/unit/services/export/test_geojson.py @@ -7,6 +7,7 @@ from kili.entrypoints.queries.label import QueriesLabel from kili.orm import Asset +from kili.services.export.format.geojson import GeoJsonExporter def test_kili_export_labels_geojson(mocker: pytest_mock.MockerFixture): @@ -22,10 +23,7 @@ def test_kili_export_labels_geojson(mocker: pytest_mock.MockerFixture): mocker.patch( "kili.services.export.format.base.get_project", return_value=get_project_return_val ) - mocker.patch( - "kili.services.export.format.base.DataConnectionsQuery.__call__", - return_value=(i for i in [{"id": "fake_data_connection_id"}]), - ) + mocker.patch.object(GeoJsonExporter, "_has_data_connection", return_value=False) mocker.patch( "kili.services.export.format.base.fetch_assets", return_value=[ diff --git a/tests/unit/services/export/test_kili.py b/tests/unit/services/export/test_kili.py index c3c7f0f76..668262383 100644 --- a/tests/unit/services/export/test_kili.py +++ b/tests/unit/services/export/test_kili.py @@ -189,6 +189,7 @@ def test_kili_exporter_convert_to_pixel_coords_pdf(mocker: pytest_mock.MockerFix def test_kili_export_labels_non_normalized_pdf(mocker: pytest_mock.MockerFixture): get_project_return_val = { "inputType": "PDF", + "dataConnections": None, "id": "fake_proj_id", "title": "fake_proj_title", "description": "fake_proj_description", @@ -273,6 +274,7 @@ def test_kili_export_labels_non_normalized_image(mocker: pytest_mock.MockerFixtu get_project_return_val = { "id": "fake_proj_id", "title": "hgfhfg", + "dataConnections": None, "inputType": "IMAGE", "jsonInterface": { "jobs": { @@ -444,6 +446,7 @@ def test_kili_export_labels_non_normalized_video(mocker: pytest_mock.MockerFixtu "title": "Object tracking on video", "description": "Use bounding-box to track objects across video frames.", "id": "fake_proj_id", + "dataConnections": None, } mocker.patch("kili.services.export.get_project", return_value=get_project_return_val) diff --git a/tests/unit/services/export/test_voc.py b/tests/unit/services/export/test_voc.py index 99a7cbdad..9d5335874 100644 --- a/tests/unit/services/export/test_voc.py +++ b/tests/unit/services/export/test_voc.py @@ -4,7 +4,10 @@ from kili.entrypoints.queries.label import QueriesLabel from kili.services.export.exceptions import NotCompatibleOptions -from kili.services.export.format.voc import _convert_from_kili_to_voc_format +from kili.services.export.format.voc import ( + VocExporter, + _convert_from_kili_to_voc_format, +) from tests.fakes.fake_data import asset_image_1, asset_image_1_without_annotation @@ -44,15 +47,13 @@ def test_when_exporting_to_voc_given_a_project_with_data_connection_then_it_shou "inputType": "IMAGE", "title": "", "id": "fake_proj_id", + "dataConnections": None, } mocker.patch("kili.services.export.get_project", return_value=get_project_return_val) mocker.patch( "kili.services.export.format.base.get_project", return_value=get_project_return_val ) - mocker.patch( - "kili.services.export.format.base.DataConnectionsQuery.__call__", - return_value=(i for i in [{"id": "fake_data_connection_id"}]), - ) + mocker.patch.object(VocExporter, "_has_data_connection", return_value=True) kili = QueriesLabel() kili.api_endpoint = "https://" # type: ignore diff --git a/tests/unit/services/export/test_yolo.py b/tests/unit/services/export/test_yolo.py index 1ea561784..15fca510c 100644 --- a/tests/unit/services/export/test_yolo.py +++ b/tests/unit/services/export/test_yolo.py @@ -10,6 +10,7 @@ from kili.entrypoints.queries.label import QueriesLabel from kili.orm import Asset from kili.services.export.format.yolo import ( + YoloExporter, _convert_from_kili_to_yolo_format, _process_asset, _write_class_file, @@ -283,6 +284,7 @@ def test_yolo_v8_merged(mocker: pytest_mock.MockerFixture): "kili.services.export.format.base.fetch_assets", return_value=[Asset(asset) for asset in assets], ) + mocker.patch.object(YoloExporter, "_has_data_connection", return_value=False) kili = QueriesLabel() kili.api_endpoint = "https://" # type: ignore @@ -334,6 +336,7 @@ def test_yolo_v8_split_jobs(mocker: pytest_mock.MockerFixture): "kili.services.export.format.base.fetch_assets", return_value=[Asset(asset) for asset in assets], ) + mocker.patch.object(YoloExporter, "_has_data_connection", return_value=False) kili = QueriesLabel() kili.api_endpoint = "https://" # type: ignore