From c7ccb2670298a8449d689b48535a29b6106ae042 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 8 Jan 2025 18:20:53 +0100 Subject: [PATCH 1/2] Issue #165 update to new `job_option` handling in `connection.create_job` --- CHANGELOG.md | 4 ++++ setup.py | 2 +- src/openeo_aggregator/backend.py | 8 +++----- tests/test_views.py | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aab517..77be526 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is roughly based on [Keep a Changelog](https://keepachangelog.com/en/ +## unreleased + +- Update to harmonized `job_option`/`additional` handling in `Connection.create_job` ([#165](https://github.com/Open-EO/openeo-aggregator/issues/165), [Open-EO/openeo-python-client#683](https://github.com/Open-EO/openeo-python-client/issues/683)) + ## 0.39.0 - More advanced process-graph splitting for cross-backend execution: not limited to splitting off `load_collection` nodes, but cut deeper into the graph. ([#150](https://github.com/Open-EO/openeo-aggregator/issues/150)) diff --git a/setup.py b/setup.py index c729182..8512b2f 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ install_requires=[ "requests", "attrs", - "openeo>=0.27.0", + "openeo>=0.36.0", "openeo_driver>=0.107.4.dev", "flask~=2.0", "gunicorn~=20.0", diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index 4a08128..d906674 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -866,13 +866,11 @@ def _create_job_standard( process_graph = self.processing.preprocess_process_graph(process_graph, backend_id=backend_id) if job_options: - additional = {k: v for k, v in job_options.items() if not k.startswith("_agg_")} - else: - additional = None + job_options = {k: v for k, v in job_options.items() if not k.startswith("_agg_")} if get_backend_config().job_options_update: # Allow fine-tuning job options through config - additional = get_backend_config().job_options_update(job_options=additional, backend_id=backend_id) + job_options = get_backend_config().job_options_update(job_options=job_options, backend_id=backend_id) con = self.backends.get_connection(backend_id) with con.authenticated_from_request(request=flask.request, user=User(user_id=user_id)), con.override( @@ -885,7 +883,7 @@ def _create_job_standard( description=metadata.get("description"), plan=metadata.get("plan"), budget=metadata.get("budget"), - additional=additional, + job_options=job_options, ) except OpenEoApiError as e: for exc_class in [ProcessGraphMissingException, ProcessGraphInvalidException]: diff --git a/tests/test_views.py b/tests/test_views.py index 34a99ec..5d61d4b 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1953,7 +1953,7 @@ def post_jobs(request: requests.Request, context): ).assert_status_code(201) assert res.headers["Location"] == f"http://oeoa.test/openeo/1.0.0/jobs/{expected}-th3j0b" assert res.headers["OpenEO-Identifier"] == f"{expected}-th3j0b" - assert jobs == [{"process": {"process_graph": pg}}] + assert jobs == [{"process": {"process_graph": pg}, "job_options": {}}] assert (backend1_post_jobs.call_count, backend2_post_jobs.call_count) == { "b1": (1, 0), From 47e4b2bd9c942cb9a479efc22698b8b4cf866211 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 8 Jan 2025 19:28:27 +0100 Subject: [PATCH 2/2] Issue #165 update to new "version" property in `/processes` introduced by Open-EO/openeo-python-driver#352 --- setup.py | 2 +- tests/test_views.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 8512b2f..ec9eed2 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ "requests", "attrs", "openeo>=0.36.0", - "openeo_driver>=0.107.4.dev", + "openeo_driver>=0.123.0.dev", "flask~=2.0", "gunicorn~=20.0", "python-json-logger>=2.0.0", diff --git a/tests/test_views.py b/tests/test_views.py index 5d61d4b..0b3051b 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -727,6 +727,7 @@ def test_processes_basic(self, api100, requests_mock, backend1, backend2): }, ], "links": [], + "version": None, } @pytest.mark.parametrize( @@ -844,7 +845,7 @@ def test_processes_resilience( else: requests_mock.get(backend2 + "/processes", status_code=404, text="nope") res = api100.get("/processes").assert_status_code(200).json - assert res == {"processes": expected, "links": []} + assert res == {"processes": expected, "links": [], "version": None} def test_result_basic_math_basic_auth(self, api100, requests_mock, backend1, backend2): def post_result(request: requests.Request, context): @@ -1006,6 +1007,7 @@ def test_processes_different_versions(self, api100, requests_mock, backend1, bac DictSubSet({"id": "prod", "federation:backends": ["b2"]}), ], "links": [], + "version": None, } def test_result_backend_by_collection_multiple_hits(self, api100, requests_mock, backend1, backend2, caplog):