From 3430c300cc9e4d1f1ffbba7d8e2dda436afd6608 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Mon, 2 Oct 2023 20:18:00 -0400 Subject: [PATCH 01/18] workflow details, if function called in jupyter environment show it as a table --- pygeoweaver/sc_detail.py | 59 ++++++++++++++++++++++++++++++++-------- pygeoweaver/utils.py | 20 ++++++++++++++ 2 files changed, 67 insertions(+), 12 deletions(-) diff --git a/pygeoweaver/sc_detail.py b/pygeoweaver/sc_detail.py index c98f89b..f8deded 100644 --- a/pygeoweaver/sc_detail.py +++ b/pygeoweaver/sc_detail.py @@ -1,9 +1,10 @@ """ Detail subcommand """ - +import json import subprocess +import pandas as pd import requests from pygeoweaver.constants import * @@ -11,24 +12,58 @@ download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, - get_root_dir, + get_root_dir, create_table, ) +import ipywidgets as widgets +from IPython.display import display, HTML def detail_workflow(workflow_id): if not workflow_id: raise RuntimeError("Workflow id is missing") download_geoweaver_jar() - subprocess.run( - [ - get_java_bin_path(), - "-jar", - get_geoweaver_jar_path(), - "detail", - f"--workflow-id={workflow_id}", - ], - cwd=f"{get_root_dir()}/", - ) + try: + # If we're in a Jupyter environment, use IPython Widgets to display the table + from IPython import get_ipython + if 'IPKernelApp' in get_ipython().config: + url = "http://localhost:8070/Geoweaver/web/detail" + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + workflow_id = "grlgu9mkbry3qe8qxdg1" + type = "workflow" + + form_data = {'type': type, 'id': workflow_id} + + d = requests.post(url=url, data=form_data, headers=headers) + d = d.json() + d['nodes'] = json.loads(d['nodes']) + + # Create an HTML table from the 'd' dictionary + table_html = create_table([d]) + + # Display the HTML table using IPython Widgets + table_output = widgets.Output() + with table_output: + display(HTML(table_html)) + + display(table_output) + except: + # If not in a Jupyter environment, use Pandas to display the table + url = "http://localhost:8070/Geoweaver/web/detail" + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + workflow_id = "grlgu9mkbry3qe8qxdg1" + type = "workflow" + + form_data = {'type': type, 'id': workflow_id} + + d = requests.post(url=url, data=form_data, headers=headers) + d = d.json() + d['nodes'] = json.loads(d['nodes']) + + # Create a Pandas DataFrame from the 'd' dictionary + df = pd.DataFrame([d]) + + # Display the DataFrame using Pandas + display(df) def detail_process(process_id): diff --git a/pygeoweaver/utils.py b/pygeoweaver/utils.py index 9e009aa..62b01b8 100644 --- a/pygeoweaver/utils.py +++ b/pygeoweaver/utils.py @@ -146,3 +146,23 @@ def copy_files(source_folder, destination_folder): ) os.makedirs(os.path.dirname(destination_file), exist_ok=True) shutil.copy2(source_file, destination_file) + + +def create_table(data): + table_html = "" + + # Create table headers + for key in data[0].keys(): + table_html += f"" + table_html += "" + + # Create table rows + for row in data: + table_html += "" + for value in row.values(): + table_html += f"" + table_html += "" + + table_html += "
{key}
{value}
" + + return table_html From 4ec0add7c0a19d366fa4d3c6f0d116329e700606 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Mon, 2 Oct 2023 20:26:09 -0400 Subject: [PATCH 02/18] optimization changes --- pygeoweaver/sc_detail.py | 41 ++++++++++------------------------------ 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/pygeoweaver/sc_detail.py b/pygeoweaver/sc_detail.py index f8deded..f57dba2 100644 --- a/pygeoweaver/sc_detail.py +++ b/pygeoweaver/sc_detail.py @@ -22,47 +22,26 @@ def detail_workflow(workflow_id): if not workflow_id: raise RuntimeError("Workflow id is missing") download_geoweaver_jar() + url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/detail" + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + workflow_id = "grlgu9mkbry3qe8qxdg1" + type = "workflow" + + form_data = {'type': type, 'id': workflow_id} + + d = requests.post(url=url, data=form_data, headers=headers) + d = d.json() + d['nodes'] = json.loads(d['nodes']) try: - # If we're in a Jupyter environment, use IPython Widgets to display the table from IPython import get_ipython if 'IPKernelApp' in get_ipython().config: - url = "http://localhost:8070/Geoweaver/web/detail" - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - workflow_id = "grlgu9mkbry3qe8qxdg1" - type = "workflow" - - form_data = {'type': type, 'id': workflow_id} - - d = requests.post(url=url, data=form_data, headers=headers) - d = d.json() - d['nodes'] = json.loads(d['nodes']) - - # Create an HTML table from the 'd' dictionary table_html = create_table([d]) - - # Display the HTML table using IPython Widgets table_output = widgets.Output() with table_output: display(HTML(table_html)) - display(table_output) except: - # If not in a Jupyter environment, use Pandas to display the table - url = "http://localhost:8070/Geoweaver/web/detail" - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - workflow_id = "grlgu9mkbry3qe8qxdg1" - type = "workflow" - - form_data = {'type': type, 'id': workflow_id} - - d = requests.post(url=url, data=form_data, headers=headers) - d = d.json() - d['nodes'] = json.loads(d['nodes']) - - # Create a Pandas DataFrame from the 'd' dictionary df = pd.DataFrame([d]) - - # Display the DataFrame using Pandas display(df) From 380c1ce099f5bc39a0c1fd1bacdd62601aa9236f Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Mon, 2 Oct 2023 20:28:08 -0400 Subject: [PATCH 03/18] changes to process detail --- pygeoweaver/sc_detail.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/pygeoweaver/sc_detail.py b/pygeoweaver/sc_detail.py index f57dba2..1d08772 100644 --- a/pygeoweaver/sc_detail.py +++ b/pygeoweaver/sc_detail.py @@ -24,11 +24,7 @@ def detail_workflow(workflow_id): download_geoweaver_jar() url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/detail" headers = {'Content-Type': 'application/x-www-form-urlencoded'} - workflow_id = "grlgu9mkbry3qe8qxdg1" - type = "workflow" - - form_data = {'type': type, 'id': workflow_id} - + form_data = {'type': 'workflow', 'id': workflow_id} d = requests.post(url=url, data=form_data, headers=headers) d = d.json() d['nodes'] = json.loads(d['nodes']) @@ -49,16 +45,23 @@ def detail_process(process_id): if not process_id: raise RuntimeError("Process id is missing") download_geoweaver_jar() - subprocess.run( - [ - get_java_bin_path(), - "-jar", - get_geoweaver_jar_path(), - "detail", - f"--process-id={process_id}", - ], - cwd=f"{get_root_dir()}/", - ) + url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/detail" + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + form_data = {'type': 'process', 'id': process_id} + d = requests.post(url=url, data=form_data, headers=headers) + d = d.json() + d['nodes'] = json.loads(d['nodes']) + try: + from IPython import get_ipython + if 'IPKernelApp' in get_ipython().config: + table_html = create_table([d]) + table_output = widgets.Output() + with table_output: + display(HTML(table_html)) + display(table_output) + except: + df = pd.DataFrame([d]) + display(df) def detail_host(host_id): From 4881ae16f331259305401bf8030bebd9180087f5 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Mon, 2 Oct 2023 20:41:18 -0400 Subject: [PATCH 04/18] changes to host --- pygeoweaver/sc_detail.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/pygeoweaver/sc_detail.py b/pygeoweaver/sc_detail.py index 1d08772..250adc9 100644 --- a/pygeoweaver/sc_detail.py +++ b/pygeoweaver/sc_detail.py @@ -37,8 +37,7 @@ def detail_workflow(workflow_id): display(HTML(table_html)) display(table_output) except: - df = pd.DataFrame([d]) - display(df) + return pd.DataFrame([d]) def detail_process(process_id): @@ -60,24 +59,28 @@ def detail_process(process_id): display(HTML(table_html)) display(table_output) except: - df = pd.DataFrame([d]) - display(df) + return pd.DataFrame([d]) def detail_host(host_id): if not host_id: raise RuntimeError("Host id is missing") download_geoweaver_jar() - subprocess.run( - [ - get_java_bin_path(), - "-jar", - get_geoweaver_jar_path(), - "detail", - f"--host-id={host_id}", - ], - cwd=f"{get_root_dir()}/", - ) + url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/detail" + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + form_data = {'type': 'host', 'id': host_id} + d = requests.post(url=url, data=form_data, headers=headers) + try: + from IPython import get_ipython + if 'IPKernelApp' in get_ipython().config: + table_html = create_table([d]) + table_output = widgets.Output() + with table_output: + display(HTML(table_html)) + display(table_output) + except: + return pd.DataFrame([d]) + def get_process_code(process_id): From 8bc13916db6a26906d640462689a4bf17448e59a Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Mon, 2 Oct 2023 20:41:28 -0400 Subject: [PATCH 05/18] format file --- pygeoweaver/sc_detail.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pygeoweaver/sc_detail.py b/pygeoweaver/sc_detail.py index 250adc9..46d14b2 100644 --- a/pygeoweaver/sc_detail.py +++ b/pygeoweaver/sc_detail.py @@ -82,7 +82,6 @@ def detail_host(host_id): return pd.DataFrame([d]) - def get_process_code(process_id): r = requests.post( f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/detail", From 26614b57a2a3d80891196247bef6e4df8499553b Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Mon, 2 Oct 2023 20:46:15 -0400 Subject: [PATCH 06/18] cleanup --- pygeoweaver/sc_create.py | 3 --- pygeoweaver/sc_detail.py | 1 - 2 files changed, 4 deletions(-) diff --git a/pygeoweaver/sc_create.py b/pygeoweaver/sc_create.py index dd1055c..5e51f90 100644 --- a/pygeoweaver/sc_create.py +++ b/pygeoweaver/sc_create.py @@ -6,9 +6,6 @@ from pygeoweaver.constants import * from pygeoweaver.utils import ( download_geoweaver_jar, - get_geoweaver_jar_path, - get_java_bin_path, - get_root_dir, check_ipython, ) diff --git a/pygeoweaver/sc_detail.py b/pygeoweaver/sc_detail.py index 46d14b2..bdf246b 100644 --- a/pygeoweaver/sc_detail.py +++ b/pygeoweaver/sc_detail.py @@ -2,7 +2,6 @@ Detail subcommand """ import json -import subprocess import pandas as pd import requests From eaa115f5ddfc5da7aaba49b072b7e2f074b77379 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Tue, 3 Oct 2023 03:52:10 -0400 Subject: [PATCH 07/18] changes to import_workflow --- pygeoweaver/sc_find.py | 6 +++--- pygeoweaver/sc_import.py | 28 +++++++++++++++++----------- pygeoweaver/sc_interface.py | 5 ++--- pygeoweaver/sc_run.py | 3 ++- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/pygeoweaver/sc_find.py b/pygeoweaver/sc_find.py index a35afc3..bb51e8f 100644 --- a/pygeoweaver/sc_find.py +++ b/pygeoweaver/sc_find.py @@ -17,7 +17,7 @@ def get_process_by_name(process_name): pd.set_option("display.max_columns", None) # Display all columns pd.set_option("display.max_rows", None) # Display all rows pd.set_option("display.expand_frame_repr", False) # Prevent truncation of columns - pd.DataFrame(matching_processes) + return pd.DataFrame(matching_processes) def get_process_by_id(process_id): @@ -34,7 +34,7 @@ def get_process_by_id(process_id): pd.set_option("display.max_columns", None) # Display all columns pd.set_option("display.max_rows", None) # Display all rows pd.set_option("display.expand_frame_repr", False) # Prevent truncation of columns - pd.DataFrame(matching_processes) + return pd.DataFrame(matching_processes) def get_process_by_language(language): @@ -51,4 +51,4 @@ def get_process_by_language(language): pd.set_option("display.max_columns", None) # Display all columns pd.set_option("display.max_rows", None) # Display all rows pd.set_option("display.expand_frame_repr", False) # Prevent truncation of columns - pd.DataFrame(matching_processes) + return pd.DataFrame(matching_processes) diff --git a/pygeoweaver/sc_import.py b/pygeoweaver/sc_import.py index aa604ad..c1b64e8 100644 --- a/pygeoweaver/sc_import.py +++ b/pygeoweaver/sc_import.py @@ -1,4 +1,9 @@ +import os.path import subprocess + +import requests + +from pygeoweaver import GEOWEAVER_DEFAULT_ENDPOINT_URL from pygeoweaver.utils import ( download_geoweaver_jar, get_geoweaver_jar_path, @@ -17,17 +22,18 @@ def import_workflow(workflow_zip_file_path): if not workflow_zip_file_path: raise RuntimeError("Workflow zip file path is missing") download_geoweaver_jar() - subprocess.run( - [ - get_java_bin_path(), - "-jar", - get_geoweaver_jar_path(), - "import", - "workflow", - workflow_zip_file_path, - ], - cwd=f"{get_root_dir()}/", - ) + file_upload_servlet = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/FileUploadServlet" + preload_url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/preload/workflow" + load_url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/load/workflow" + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + file_name = os.path.basename(workflow_zip_file_path) + _id, _ = os.path.splitext(file_name) + files = {'file': (file_name, open(workflow_zip_file_path, 'rb'))} + requests.post(url=file_upload_servlet, files=files) # upload + requests.post(url=preload_url, headers=headers, data={'id': _id, 'filename': file_name}) # preload + requests.post(url=load_url, headers=headers, data={'id': _id, 'filename': file_name}) # load + return 'Import success.' + def import_workflow_from_github(git_repo_url): pass diff --git a/pygeoweaver/sc_interface.py b/pygeoweaver/sc_interface.py index e17b569..9239771 100644 --- a/pygeoweaver/sc_interface.py +++ b/pygeoweaver/sc_interface.py @@ -6,11 +6,10 @@ from pygeoweaver.sc_export import * from pygeoweaver.sc_history import * from pygeoweaver.sc_import import * -from pygeoweaver.sc_list import * from pygeoweaver.sc_run import * from pygeoweaver.server import * from pygeoweaver.sc_resetpassword import * from pygeoweaver.sc_help import * from pygeoweaver.sc_create import * -from pygeoweaver.sc_sync import * -from pygeoweaver.sc_find import * +from pygeoweaver.sc_sync import * # +from pygeoweaver.sc_find import * # diff --git a/pygeoweaver/sc_run.py b/pygeoweaver/sc_run.py index 3bdd79b..e72cc52 100644 --- a/pygeoweaver/sc_run.py +++ b/pygeoweaver/sc_run.py @@ -66,7 +66,7 @@ def run_process( process_id, ], cwd=f"{get_root_dir()}/", - ) + ) # TODO: need to re-implement password encryption same as Geoweaver codebase to change this def run_workflow( @@ -171,3 +171,4 @@ def run_workflow( if environment_list: command.extend(["-e", environment_list]) subprocess.run(command, cwd=f"{get_root_dir()}/") + # TODO: need to re-implement password encryption same as Geoweaver codebase to change this From 09321ef5ec3feac7113a9b4bf2bb0682eaa0534b Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Tue, 3 Oct 2023 03:53:33 -0400 Subject: [PATCH 08/18] cleanup --- pygeoweaver/sc_import.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pygeoweaver/sc_import.py b/pygeoweaver/sc_import.py index c1b64e8..d7e747a 100644 --- a/pygeoweaver/sc_import.py +++ b/pygeoweaver/sc_import.py @@ -5,10 +5,7 @@ from pygeoweaver import GEOWEAVER_DEFAULT_ENDPOINT_URL from pygeoweaver.utils import ( - download_geoweaver_jar, - get_geoweaver_jar_path, - get_java_bin_path, - get_root_dir, + download_geoweaver_jar ) From ad2a4c6e808da0f3ddc887e75e89091f8b94ea6a Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Tue, 3 Oct 2023 04:16:14 -0400 Subject: [PATCH 09/18] changes to list commands to use spring apis --- pygeoweaver/sc_list.py | 60 ++++++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/pygeoweaver/sc_list.py b/pygeoweaver/sc_list.py index deb6934..d55f8ac 100644 --- a/pygeoweaver/sc_list.py +++ b/pygeoweaver/sc_list.py @@ -2,32 +2,55 @@ import requests import subprocess + +from IPython.core.display_functions import display +from ipywidgets import HTML + from pygeoweaver.constants import * from pygeoweaver.utils import ( download_geoweaver_jar, get_geoweaver_jar_path, get_java_bin_path, get_root_dir, - check_ipython, + check_ipython, create_table, ) import pandas as pd def list_hosts(): download_geoweaver_jar() - subprocess.run( - [get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "list", "--host"], - cwd=f"{get_root_dir()}/", - ) + url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/list" + form_data = {'type': 'host'} + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + data = requests.post(url=url, headers=headers, data=form_data) + + if 'get_ipython' in globals(): + # Running in a Jupyter Notebook, display as HTML table + data_json = data.json() + table_html, _ = create_table(data_json) + display(HTML(table_html)) + else: + # Not running in a Jupyter Notebook, display as Pandas DataFrame + data_json = data.json() + df = pd.DataFrame(data_json) + return df def list_processes(): download_geoweaver_jar() - subprocess.run(["chmod", "+x", get_geoweaver_jar_path()], cwd=f"{get_root_dir()}/") - subprocess.run( - [get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "list", "--process"], - cwd=f"{get_root_dir()}/", - ) + url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/list" + form_data = {'type': 'process'} + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + data = requests.post(url=url, headers=headers, data=form_data) + + if 'get_ipython' in globals(): + data_json = data.json() + table_html, _ = create_table(data_json) + display(HTML(table_html)) + else: + data_json = data.json() + df = pd.DataFrame(data_json) + return df def list_processes_in_workflow(workflow_id): @@ -48,7 +71,16 @@ def list_processes_in_workflow(workflow_id): def list_workflows(): download_geoweaver_jar() - subprocess.run( - [get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "list", "--workflow"], - cwd=f"{get_root_dir()}/", - ) + url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/list" + form_data = {'type': 'workflow'} + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + data = requests.post(url=url, headers=headers, data=form_data) + + if 'get_ipython' in globals(): + data_json = data.json() + table_html, _ = create_table(data_json) + display(HTML(table_html)) + else: + data_json = data.json() + df = pd.DataFrame(data_json) + return df From e4021d22394ba8a7363e6e82bd24958d853a0b56 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Tue, 3 Oct 2023 17:03:35 -0400 Subject: [PATCH 10/18] added test_detail for changes in test detail --- pygeoweaver/sc_import.py | 6 ++-- pygeoweaver/sc_interface.py | 4 +-- test/test_detail.py | 64 ++++++++++++++++++++++++++----------- 3 files changed, 49 insertions(+), 25 deletions(-) diff --git a/pygeoweaver/sc_import.py b/pygeoweaver/sc_import.py index d7e747a..c6bda9a 100644 --- a/pygeoweaver/sc_import.py +++ b/pygeoweaver/sc_import.py @@ -1,11 +1,8 @@ import os.path -import subprocess import requests - -from pygeoweaver import GEOWEAVER_DEFAULT_ENDPOINT_URL from pygeoweaver.utils import ( - download_geoweaver_jar + download_geoweaver_jar, ) @@ -16,6 +13,7 @@ def import_workflow(workflow_zip_file_path): Geoweaver workflow zip file path """ + from pygeoweaver import GEOWEAVER_DEFAULT_ENDPOINT_URL if not workflow_zip_file_path: raise RuntimeError("Workflow zip file path is missing") download_geoweaver_jar() diff --git a/pygeoweaver/sc_interface.py b/pygeoweaver/sc_interface.py index 9239771..f2c769c 100644 --- a/pygeoweaver/sc_interface.py +++ b/pygeoweaver/sc_interface.py @@ -11,5 +11,5 @@ from pygeoweaver.sc_resetpassword import * from pygeoweaver.sc_help import * from pygeoweaver.sc_create import * -from pygeoweaver.sc_sync import * # -from pygeoweaver.sc_find import * # +from pygeoweaver.sc_sync import * +from pygeoweaver.sc_find import * diff --git a/test/test_detail.py b/test/test_detail.py index dc9bd25..5b6d222 100644 --- a/test/test_detail.py +++ b/test/test_detail.py @@ -1,29 +1,55 @@ -from io import StringIO -import sys +import json import unittest -from pygeoweaver.sc_detail import detail_host, detail_process, detail_workflow -from pygeoweaver.utils import get_logger +import pandas as pd +from unittest.mock import patch, Mock +from pygeoweaver import get_logger logger = get_logger(__name__) -def test_detail_process(capfd): - detail_process("not_existing_id") - output, err = capfd.readouterr() - logger.debug("stdout_output" + output) - assert "No process found with id: not_existing_id" in output +class TestDetailFunctions(unittest.TestCase): + @patch('pygeoweaver.utils.download_geoweaver_jar') + @patch('requests.post') + def test_detail_workflow(self, mock_post, mock_download): + from pygeoweaver.sc_detail import detail_workflow + with self.assertRaises(RuntimeError): + detail_workflow(None) + mock_response = Mock() + mock_response.json.return_value = {'nodes': json.dumps({"some_key": "some_value"})} + mock_post.return_value = mock_response + result = detail_workflow('some_id') + self.assertIsInstance(result, pd.DataFrame) -def test_detail_workflow(capfd): - detail_workflow("not_existing_id") - output, err = capfd.readouterr() - logger.debug("stdout_output" + output) - assert "No workflow found with id: not_existing_id" in output + @patch('pygeoweaver.utils.download_geoweaver_jar') + @patch('requests.post') + def test_detail_process(self, mock_post, mock_download): + from pygeoweaver.sc_detail import detail_process + with self.assertRaises(RuntimeError): + detail_process(None) + mock_response = Mock() + mock_response.json.return_value = {'nodes': json.dumps({"some_key": "some_value"})} + mock_post.return_value = mock_response + result = detail_process('some_id') + self.assertIsInstance(result, pd.DataFrame) -def test_detail_host(capfd): - detail_host("not_existing_id") - output, err = capfd.readouterr() - logger.debug("stdout_output" + output) - assert "No host found with id: not_existing_id" in output + @patch('pygeoweaver.utils.download_geoweaver_jar') + @patch('requests.post') + def test_detail_host(self, mock_post, mock_download): + from pygeoweaver.sc_detail import detail_host + + with self.assertRaises(RuntimeError): + detail_host(None) + + mock_response = Mock() + mock_response.json.return_value = {"some_key": "some_value"} + mock_post.return_value = mock_response + + result = detail_host('some_id') + self.assertIsInstance(result, pd.DataFrame) + + +if __name__ == '__main__': + unittest.main() From 959034e78d2d9ffa7737b4c604b6cbbcde8dcf9f Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Tue, 3 Oct 2023 17:08:21 -0400 Subject: [PATCH 11/18] added test_import for import workflow functionality --- test/test_import.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 test/test_import.py diff --git a/test/test_import.py b/test/test_import.py new file mode 100644 index 0000000..87aad21 --- /dev/null +++ b/test/test_import.py @@ -0,0 +1,35 @@ +import os +import unittest +from unittest.mock import patch, Mock +from pygeoweaver.sc_import import import_workflow + + +class TestImportWorkflow(unittest.TestCase): + + @patch('os.path.basename') + @patch('os.path.splitext') + @patch('builtins.open') + @patch('requests.post') + def test_import_workflow(self, mock_post, mock_open, mock_splitext, mock_basename): + # Setup + mock_basename.return_value = 'test_file.zip' + mock_splitext.return_value = ('test_file', '.zip') + mock_open.return_value = Mock() + + mock_response = Mock() + mock_response.json.return_value = {'status': 'success'} + mock_post.return_value = mock_response + + # Mocking GEOWEAVER_DEFAULT_ENDPOINT_URL + with patch('pygeoweaver.constants.GEOWEAVER_DEFAULT_ENDPOINT_URL', 'http://example.com'): + # Run + result = import_workflow('/path/to/test_file.zip') + + # Asserts + self.assertEqual(result, 'Import success.') + mock_post.assert_called() + mock_open.assert_called_with('/path/to/test_file.zip', 'rb') + + +if __name__ == '__main__': + unittest.main() From c4533e980e43f1591e3afafccaf296e3717f8c37 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Tue, 3 Oct 2023 18:30:29 -0400 Subject: [PATCH 12/18] added test cases for sc_list --- pygeoweaver/sc_list.py | 16 ++++++------ test/test_list.py | 56 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 test/test_list.py diff --git a/pygeoweaver/sc_list.py b/pygeoweaver/sc_list.py index d55f8ac..d2c2f47 100644 --- a/pygeoweaver/sc_list.py +++ b/pygeoweaver/sc_list.py @@ -1,7 +1,6 @@ import json import requests -import subprocess from IPython.core.display_functions import display from ipywidgets import HTML @@ -9,22 +8,21 @@ from pygeoweaver.constants import * from pygeoweaver.utils import ( download_geoweaver_jar, - get_geoweaver_jar_path, - get_java_bin_path, - get_root_dir, check_ipython, create_table, ) import pandas as pd def list_hosts(): + from IPython import get_ipython + ip = get_ipython() download_geoweaver_jar() url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/list" form_data = {'type': 'host'} headers = {'Content-Type': 'application/x-www-form-urlencoded'} data = requests.post(url=url, headers=headers, data=form_data) - if 'get_ipython' in globals(): + if ip is not None: # Running in a Jupyter Notebook, display as HTML table data_json = data.json() table_html, _ = create_table(data_json) @@ -37,13 +35,15 @@ def list_hosts(): def list_processes(): + from IPython import get_ipython + ip = get_ipython() download_geoweaver_jar() url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/list" form_data = {'type': 'process'} headers = {'Content-Type': 'application/x-www-form-urlencoded'} data = requests.post(url=url, headers=headers, data=form_data) - if 'get_ipython' in globals(): + if ip is not None: data_json = data.json() table_html, _ = create_table(data_json) display(HTML(table_html)) @@ -70,13 +70,15 @@ def list_processes_in_workflow(workflow_id): def list_workflows(): + from IPython import get_ipython + ip = get_ipython() download_geoweaver_jar() url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/list" form_data = {'type': 'workflow'} headers = {'Content-Type': 'application/x-www-form-urlencoded'} data = requests.post(url=url, headers=headers, data=form_data) - if 'get_ipython' in globals(): + if ip is not None: data_json = data.json() table_html, _ = create_table(data_json) display(HTML(table_html)) diff --git a/test/test_list.py b/test/test_list.py new file mode 100644 index 0000000..9cd2dd9 --- /dev/null +++ b/test/test_list.py @@ -0,0 +1,56 @@ +import unittest +from unittest.mock import patch, MagicMock +import pygeoweaver # replace 'your_module' with the actual module name where pygeoweaver is defined + + +class TestListHosts(unittest.TestCase): + @patch('IPython.core.display_functions.display') + @patch('requests.post') + def test_list_hosts(self, mock_post, mock_display): + mock_post.return_value.json.return_value = [{'some': 'data'}] + pygeoweaver.create_table = MagicMock(return_value=('table_html', 'some_other_value')) + + with patch('builtins.globals', return_value={'get_ipython': None}): + pygeoweaver.list_hosts() + + +class TestListProcesses(unittest.TestCase): + + @patch('IPython.core.display_functions.display') + @patch('requests.post') + def test_list_processes(self, mock_post, mock_display): + mock_post.return_value.json.return_value = [{'some': 'data'}] + pygeoweaver.create_table = MagicMock(return_value=('table_html', None)) + + with patch('builtins.globals', return_value={'get_ipython': None}): + pygeoweaver.list_processes() + + +class TestListProcessesInWorkflow(unittest.TestCase): + + @patch('json.loads') + @patch('requests.post') + def test_list_processes_in_workflow(self, mock_post, mock_json_loads): + mock_post.return_value.json.return_value = {'nodes': '[{"title": "some_title", "id": "some_id"}]'} + mock_json_loads.return_value = [{'title': 'some_title', 'id': 'some_id'}] + + result = pygeoweaver.list_processes_in_workflow('some_workflow_id') + + expected_result = [{'title': 'some_title', 'id': 'some_id'}] + self.assertEqual(result, expected_result) + + +class TestListWorkflows(unittest.TestCase): + + @patch('IPython.core.display_functions.display') + @patch('requests.post') + def test_list_workflows(self, mock_post, mock_display): + mock_post.return_value.json.return_value = [{'some': 'data'}] + pygeoweaver.create_table = MagicMock(return_value=('table_html', None)) + + with patch('builtins.globals', return_value={'get_ipython': None}): + pygeoweaver.list_workflows() + + +if __name__ == '__main__': + unittest.main() From f34cd1f04368a0383e76e4630374580ac38f13bb Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Tue, 3 Oct 2023 20:48:00 -0400 Subject: [PATCH 13/18] fix correct import statements --- pygeoweaver/sc_list.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pygeoweaver/sc_list.py b/pygeoweaver/sc_list.py index d2c2f47..1a1e7be 100644 --- a/pygeoweaver/sc_list.py +++ b/pygeoweaver/sc_list.py @@ -2,8 +2,7 @@ import requests -from IPython.core.display_functions import display -from ipywidgets import HTML +from IPython.core.display import display, HTML from pygeoweaver.constants import * from pygeoweaver.utils import ( @@ -25,8 +24,8 @@ def list_hosts(): if ip is not None: # Running in a Jupyter Notebook, display as HTML table data_json = data.json() - table_html, _ = create_table(data_json) - display(HTML(table_html)) + table_html = create_table(data_json) + return display(HTML(table_html)) else: # Not running in a Jupyter Notebook, display as Pandas DataFrame data_json = data.json() @@ -45,7 +44,7 @@ def list_processes(): if ip is not None: data_json = data.json() - table_html, _ = create_table(data_json) + table_html = create_table(data_json) display(HTML(table_html)) else: data_json = data.json() @@ -80,7 +79,7 @@ def list_workflows(): if ip is not None: data_json = data.json() - table_html, _ = create_table(data_json) + table_html = create_table(data_json) display(HTML(table_html)) else: data_json = data.json() From 02df3d8d4a95b3d0f7b019d200a717eed603edaf Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Tue, 3 Oct 2023 21:06:33 -0400 Subject: [PATCH 14/18] final testing changes and import optimizations --- pygeoweaver/sc_detail.py | 7 ++----- pygeoweaver/utils.py | 25 +++++++++++++++---------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/pygeoweaver/sc_detail.py b/pygeoweaver/sc_detail.py index bdf246b..c8e57ed 100644 --- a/pygeoweaver/sc_detail.py +++ b/pygeoweaver/sc_detail.py @@ -9,9 +9,7 @@ from pygeoweaver.utils import ( download_geoweaver_jar, - get_geoweaver_jar_path, - get_java_bin_path, - get_root_dir, create_table, + create_table, ) import ipywidgets as widgets from IPython.display import display, HTML @@ -48,7 +46,6 @@ def detail_process(process_id): form_data = {'type': 'process', 'id': process_id} d = requests.post(url=url, data=form_data, headers=headers) d = d.json() - d['nodes'] = json.loads(d['nodes']) try: from IPython import get_ipython if 'IPKernelApp' in get_ipython().config: @@ -72,7 +69,7 @@ def detail_host(host_id): try: from IPython import get_ipython if 'IPKernelApp' in get_ipython().config: - table_html = create_table([d]) + table_html = create_table([d.json()]) table_output = widgets.Output() with table_output: display(HTML(table_html)) diff --git a/pygeoweaver/utils.py b/pygeoweaver/utils.py index 62b01b8..478484d 100644 --- a/pygeoweaver/utils.py +++ b/pygeoweaver/utils.py @@ -148,21 +148,26 @@ def copy_files(source_folder, destination_folder): shutil.copy2(source_file, destination_file) -def create_table(data): +def create_table(data, max_length=100): table_html = "" # Create table headers - for key in data[0].keys(): - table_html += f"" - table_html += "" - - # Create table rows - for row in data: - table_html += "" - for value in row.values(): - table_html += f"" + if len(data) > 0: + for key in data[0].keys(): + table_html += f"" table_html += "" + # Create table rows + for row in data: + table_html += "" + for value in row.values(): + # Truncate and add ellipses if the value is too long + display_value = str(value)[:max_length] + '...' if len(str(value)) > max_length else str(value) + table_html += f"" + table_html += "" + else: + table_html += "" + table_html += "
{key}
{value}{key}
{display_value}
No Data
" return table_html From 5ec3e7486eda9b09dd8a6147059778bf75b5b003 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Tue, 3 Oct 2023 21:08:42 -0400 Subject: [PATCH 15/18] final testing changes and import optimizations --- pygeoweaver/sc_detail.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygeoweaver/sc_detail.py b/pygeoweaver/sc_detail.py index c8e57ed..edc7602 100644 --- a/pygeoweaver/sc_detail.py +++ b/pygeoweaver/sc_detail.py @@ -65,11 +65,11 @@ def detail_host(host_id): url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/detail" headers = {'Content-Type': 'application/x-www-form-urlencoded'} form_data = {'type': 'host', 'id': host_id} - d = requests.post(url=url, data=form_data, headers=headers) + d = requests.post(url=url, data=form_data, headers=headers).json() try: from IPython import get_ipython if 'IPKernelApp' in get_ipython().config: - table_html = create_table([d.json()]) + table_html = create_table([d]) table_output = widgets.Output() with table_output: display(HTML(table_html)) From e8a981766ce9eb6d3b1a7abb9660e21578cd8f33 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Wed, 4 Oct 2023 15:31:33 -0400 Subject: [PATCH 16/18] changes to get_detail func --- pygeoweaver/sc_detail.py | 58 +++------------------------------------- pygeoweaver/utils.py | 29 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 54 deletions(-) diff --git a/pygeoweaver/sc_detail.py b/pygeoweaver/sc_detail.py index edc7602..fbaeb26 100644 --- a/pygeoweaver/sc_detail.py +++ b/pygeoweaver/sc_detail.py @@ -10,72 +10,22 @@ from pygeoweaver.utils import ( download_geoweaver_jar, create_table, + get_detail ) import ipywidgets as widgets from IPython.display import display, HTML def detail_workflow(workflow_id): - if not workflow_id: - raise RuntimeError("Workflow id is missing") - download_geoweaver_jar() - url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/detail" - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - form_data = {'type': 'workflow', 'id': workflow_id} - d = requests.post(url=url, data=form_data, headers=headers) - d = d.json() - d['nodes'] = json.loads(d['nodes']) - try: - from IPython import get_ipython - if 'IPKernelApp' in get_ipython().config: - table_html = create_table([d]) - table_output = widgets.Output() - with table_output: - display(HTML(table_html)) - display(table_output) - except: - return pd.DataFrame([d]) + return get_detail(workflow_id, 'workflow') def detail_process(process_id): - if not process_id: - raise RuntimeError("Process id is missing") - download_geoweaver_jar() - url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/detail" - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - form_data = {'type': 'process', 'id': process_id} - d = requests.post(url=url, data=form_data, headers=headers) - d = d.json() - try: - from IPython import get_ipython - if 'IPKernelApp' in get_ipython().config: - table_html = create_table([d]) - table_output = widgets.Output() - with table_output: - display(HTML(table_html)) - display(table_output) - except: - return pd.DataFrame([d]) + return get_detail(process_id, 'process') def detail_host(host_id): - if not host_id: - raise RuntimeError("Host id is missing") - download_geoweaver_jar() - url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/detail" - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - form_data = {'type': 'host', 'id': host_id} - d = requests.post(url=url, data=form_data, headers=headers).json() - try: - from IPython import get_ipython - if 'IPKernelApp' in get_ipython().config: - table_html = create_table([d]) - table_output = widgets.Output() - with table_output: - display(HTML(table_html)) - display(table_output) - except: - return pd.DataFrame([d]) + return get_detail(host_id, 'host') def get_process_code(process_id): diff --git a/pygeoweaver/utils.py b/pygeoweaver/utils.py index 478484d..111a50d 100644 --- a/pygeoweaver/utils.py +++ b/pygeoweaver/utils.py @@ -1,12 +1,19 @@ +import json import os import sys import shutil import logging import subprocess + +import pandas as pd import requests import platform from IPython import get_ipython +import ipywidgets as widgets +from IPython.display import display, HTML + +from pygeoweaver import GEOWEAVER_DEFAULT_ENDPOINT_URL def get_home_dir(): @@ -171,3 +178,25 @@ def create_table(data, max_length=100): table_html += "" return table_html + + +def get_detail(id, type): + if not id: + raise RuntimeError("Workflow id is missing") + download_geoweaver_jar() + url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/detail" + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + form_data = {'type': type, 'id': id} + d = requests.post(url=url, data=form_data, headers=headers) + d = d.json() + d['nodes'] = json.loads(d['nodes']) + try: + from IPython import get_ipython + if 'IPKernelApp' in get_ipython().config: + table_html = create_table([d]) + table_output = widgets.Output() + with table_output: + display(HTML(table_html)) + display(table_output) + except: + return pd.DataFrame([d]) From b99f19ced008b54dcdd3791f1e9c4d49bcd57610 Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Wed, 4 Oct 2023 15:31:47 -0400 Subject: [PATCH 17/18] cleanup --- pygeoweaver/sc_detail.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pygeoweaver/sc_detail.py b/pygeoweaver/sc_detail.py index fbaeb26..76365ed 100644 --- a/pygeoweaver/sc_detail.py +++ b/pygeoweaver/sc_detail.py @@ -1,19 +1,12 @@ """ Detail subcommand """ -import json - -import pandas as pd import requests from pygeoweaver.constants import * from pygeoweaver.utils import ( - download_geoweaver_jar, - create_table, get_detail ) -import ipywidgets as widgets -from IPython.display import display, HTML def detail_workflow(workflow_id): From 5242736b40a79798427141f0d8eb4ec8ca334fee Mon Sep 17 00:00:00 2001 From: Gokul Prathin Date: Mon, 27 Nov 2023 02:29:35 -0500 Subject: [PATCH 18/18] fixed circular import issue --- pygeoweaver/interactive/__init__.py | 0 pygeoweaver/interactive/list.py | 0 pygeoweaver/sc_detail.py | 2 +- pygeoweaver/utils.py | 6 ++++-- 4 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 pygeoweaver/interactive/__init__.py create mode 100644 pygeoweaver/interactive/list.py diff --git a/pygeoweaver/interactive/__init__.py b/pygeoweaver/interactive/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pygeoweaver/interactive/list.py b/pygeoweaver/interactive/list.py new file mode 100644 index 0000000..e69de29 diff --git a/pygeoweaver/sc_detail.py b/pygeoweaver/sc_detail.py index 76365ed..a014d05 100644 --- a/pygeoweaver/sc_detail.py +++ b/pygeoweaver/sc_detail.py @@ -2,7 +2,6 @@ Detail subcommand """ import requests -from pygeoweaver.constants import * from pygeoweaver.utils import ( get_detail @@ -22,6 +21,7 @@ def detail_host(host_id): def get_process_code(process_id): + from pygeoweaver.constants import GEOWEAVER_DEFAULT_ENDPOINT_URL r = requests.post( f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/detail", data={"type": "process", "id": process_id}, diff --git a/pygeoweaver/utils.py b/pygeoweaver/utils.py index 111a50d..9996bd7 100644 --- a/pygeoweaver/utils.py +++ b/pygeoweaver/utils.py @@ -13,8 +13,6 @@ import ipywidgets as widgets from IPython.display import display, HTML -from pygeoweaver import GEOWEAVER_DEFAULT_ENDPOINT_URL - def get_home_dir(): return os.path.expanduser("~") @@ -181,6 +179,10 @@ def create_table(data, max_length=100): def get_detail(id, type): + + + from pygeoweaver.constants import GEOWEAVER_DEFAULT_ENDPOINT_URL + if not id: raise RuntimeError("Workflow id is missing") download_geoweaver_jar()