diff --git a/pygeoweaver/interactive/__init__.py b/pygeoweaver/interactive/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pygeoweaver/interactive/list.py b/pygeoweaver/interactive/list.py new file mode 100644 index 0000000..e69de29 diff --git a/pygeoweaver/sc_create.py b/pygeoweaver/sc_create.py index dd1055c..5e51f90 100644 --- a/pygeoweaver/sc_create.py +++ b/pygeoweaver/sc_create.py @@ -6,9 +6,6 @@ from pygeoweaver.constants import * from pygeoweaver.utils import ( download_geoweaver_jar, - get_geoweaver_jar_path, - get_java_bin_path, - get_root_dir, check_ipython, ) diff --git a/pygeoweaver/sc_detail.py b/pygeoweaver/sc_detail.py index c98f89b..a014d05 100644 --- a/pygeoweaver/sc_detail.py +++ b/pygeoweaver/sc_detail.py @@ -1,69 +1,27 @@ """ Detail subcommand """ - -import subprocess - import requests -from pygeoweaver.constants import * from pygeoweaver.utils import ( - download_geoweaver_jar, - get_geoweaver_jar_path, - get_java_bin_path, - get_root_dir, + get_detail ) def detail_workflow(workflow_id): - if not workflow_id: - raise RuntimeError("Workflow id is missing") - download_geoweaver_jar() - subprocess.run( - [ - get_java_bin_path(), - "-jar", - get_geoweaver_jar_path(), - "detail", - f"--workflow-id={workflow_id}", - ], - cwd=f"{get_root_dir()}/", - ) + return get_detail(workflow_id, 'workflow') def detail_process(process_id): - if not process_id: - raise RuntimeError("Process id is missing") - download_geoweaver_jar() - subprocess.run( - [ - get_java_bin_path(), - "-jar", - get_geoweaver_jar_path(), - "detail", - f"--process-id={process_id}", - ], - cwd=f"{get_root_dir()}/", - ) + return get_detail(process_id, 'process') def detail_host(host_id): - if not host_id: - raise RuntimeError("Host id is missing") - download_geoweaver_jar() - subprocess.run( - [ - get_java_bin_path(), - "-jar", - get_geoweaver_jar_path(), - "detail", - f"--host-id={host_id}", - ], - cwd=f"{get_root_dir()}/", - ) + return get_detail(host_id, 'host') def get_process_code(process_id): + from pygeoweaver.constants import GEOWEAVER_DEFAULT_ENDPOINT_URL r = requests.post( f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/detail", data={"type": "process", "id": process_id}, diff --git a/pygeoweaver/sc_find.py b/pygeoweaver/sc_find.py index a35afc3..bb51e8f 100644 --- a/pygeoweaver/sc_find.py +++ b/pygeoweaver/sc_find.py @@ -17,7 +17,7 @@ def get_process_by_name(process_name): pd.set_option("display.max_columns", None) # Display all columns pd.set_option("display.max_rows", None) # Display all rows pd.set_option("display.expand_frame_repr", False) # Prevent truncation of columns - pd.DataFrame(matching_processes) + return pd.DataFrame(matching_processes) def get_process_by_id(process_id): @@ -34,7 +34,7 @@ def get_process_by_id(process_id): pd.set_option("display.max_columns", None) # Display all columns pd.set_option("display.max_rows", None) # Display all rows pd.set_option("display.expand_frame_repr", False) # Prevent truncation of columns - pd.DataFrame(matching_processes) + return pd.DataFrame(matching_processes) def get_process_by_language(language): @@ -51,4 +51,4 @@ def get_process_by_language(language): pd.set_option("display.max_columns", None) # Display all columns pd.set_option("display.max_rows", None) # Display all rows pd.set_option("display.expand_frame_repr", False) # Prevent truncation of columns - pd.DataFrame(matching_processes) + return pd.DataFrame(matching_processes) diff --git a/pygeoweaver/sc_import.py b/pygeoweaver/sc_import.py index aa604ad..c6bda9a 100644 --- a/pygeoweaver/sc_import.py +++ b/pygeoweaver/sc_import.py @@ -1,9 +1,8 @@ -import subprocess +import os.path + +import requests from pygeoweaver.utils import ( download_geoweaver_jar, - get_geoweaver_jar_path, - get_java_bin_path, - get_root_dir, ) @@ -14,20 +13,22 @@ def import_workflow(workflow_zip_file_path): Geoweaver workflow zip file path """ + from pygeoweaver import GEOWEAVER_DEFAULT_ENDPOINT_URL if not workflow_zip_file_path: raise RuntimeError("Workflow zip file path is missing") download_geoweaver_jar() - subprocess.run( - [ - get_java_bin_path(), - "-jar", - get_geoweaver_jar_path(), - "import", - "workflow", - workflow_zip_file_path, - ], - cwd=f"{get_root_dir()}/", - ) + file_upload_servlet = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/FileUploadServlet" + preload_url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/preload/workflow" + load_url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/load/workflow" + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + file_name = os.path.basename(workflow_zip_file_path) + _id, _ = os.path.splitext(file_name) + files = {'file': (file_name, open(workflow_zip_file_path, 'rb'))} + requests.post(url=file_upload_servlet, files=files) # upload + requests.post(url=preload_url, headers=headers, data={'id': _id, 'filename': file_name}) # preload + requests.post(url=load_url, headers=headers, data={'id': _id, 'filename': file_name}) # load + return 'Import success.' + def import_workflow_from_github(git_repo_url): pass diff --git a/pygeoweaver/sc_interface.py b/pygeoweaver/sc_interface.py index e17b569..f2c769c 100644 --- a/pygeoweaver/sc_interface.py +++ b/pygeoweaver/sc_interface.py @@ -6,7 +6,6 @@ from pygeoweaver.sc_export import * from pygeoweaver.sc_history import * from pygeoweaver.sc_import import * -from pygeoweaver.sc_list import * from pygeoweaver.sc_run import * from pygeoweaver.server import * from pygeoweaver.sc_resetpassword import * diff --git a/pygeoweaver/sc_list.py b/pygeoweaver/sc_list.py index deb6934..1a1e7be 100644 --- a/pygeoweaver/sc_list.py +++ b/pygeoweaver/sc_list.py @@ -1,33 +1,55 @@ import json import requests -import subprocess + +from IPython.core.display import display, HTML + from pygeoweaver.constants import * from pygeoweaver.utils import ( download_geoweaver_jar, - get_geoweaver_jar_path, - get_java_bin_path, - get_root_dir, - check_ipython, + check_ipython, create_table, ) import pandas as pd def list_hosts(): + from IPython import get_ipython + ip = get_ipython() download_geoweaver_jar() - subprocess.run( - [get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "list", "--host"], - cwd=f"{get_root_dir()}/", - ) + url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/list" + form_data = {'type': 'host'} + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + data = requests.post(url=url, headers=headers, data=form_data) + + if ip is not None: + # Running in a Jupyter Notebook, display as HTML table + data_json = data.json() + table_html = create_table(data_json) + return display(HTML(table_html)) + else: + # Not running in a Jupyter Notebook, display as Pandas DataFrame + data_json = data.json() + df = pd.DataFrame(data_json) + return df def list_processes(): + from IPython import get_ipython + ip = get_ipython() download_geoweaver_jar() - subprocess.run(["chmod", "+x", get_geoweaver_jar_path()], cwd=f"{get_root_dir()}/") - subprocess.run( - [get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "list", "--process"], - cwd=f"{get_root_dir()}/", - ) + url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/list" + form_data = {'type': 'process'} + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + data = requests.post(url=url, headers=headers, data=form_data) + + if ip is not None: + data_json = data.json() + table_html = create_table(data_json) + display(HTML(table_html)) + else: + data_json = data.json() + df = pd.DataFrame(data_json) + return df def list_processes_in_workflow(workflow_id): @@ -47,8 +69,19 @@ def list_processes_in_workflow(workflow_id): def list_workflows(): + from IPython import get_ipython + ip = get_ipython() download_geoweaver_jar() - subprocess.run( - [get_java_bin_path(), "-jar", get_geoweaver_jar_path(), "list", "--workflow"], - cwd=f"{get_root_dir()}/", - ) + url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/list" + form_data = {'type': 'workflow'} + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + data = requests.post(url=url, headers=headers, data=form_data) + + if ip is not None: + data_json = data.json() + table_html = create_table(data_json) + display(HTML(table_html)) + else: + data_json = data.json() + df = pd.DataFrame(data_json) + return df diff --git a/pygeoweaver/sc_run.py b/pygeoweaver/sc_run.py index 3bdd79b..e72cc52 100644 --- a/pygeoweaver/sc_run.py +++ b/pygeoweaver/sc_run.py @@ -66,7 +66,7 @@ def run_process( process_id, ], cwd=f"{get_root_dir()}/", - ) + ) # TODO: need to re-implement password encryption same as Geoweaver codebase to change this def run_workflow( @@ -171,3 +171,4 @@ def run_workflow( if environment_list: command.extend(["-e", environment_list]) subprocess.run(command, cwd=f"{get_root_dir()}/") + # TODO: need to re-implement password encryption same as Geoweaver codebase to change this diff --git a/pygeoweaver/utils.py b/pygeoweaver/utils.py index 9e009aa..9996bd7 100644 --- a/pygeoweaver/utils.py +++ b/pygeoweaver/utils.py @@ -1,12 +1,17 @@ +import json import os import sys import shutil import logging import subprocess + +import pandas as pd import requests import platform from IPython import get_ipython +import ipywidgets as widgets +from IPython.display import display, HTML def get_home_dir(): @@ -146,3 +151,54 @@ def copy_files(source_folder, destination_folder): ) os.makedirs(os.path.dirname(destination_file), exist_ok=True) shutil.copy2(source_file, destination_file) + + +def create_table(data, max_length=100): + table_html = "" + + # Create table headers + if len(data) > 0: + for key in data[0].keys(): + table_html += f"" + table_html += "" + + # Create table rows + for row in data: + table_html += "" + for value in row.values(): + # Truncate and add ellipses if the value is too long + display_value = str(value)[:max_length] + '...' if len(str(value)) > max_length else str(value) + table_html += f"" + table_html += "" + else: + table_html += "" + + table_html += "
{key}
{display_value}
No Data
" + + return table_html + + +def get_detail(id, type): + + + from pygeoweaver.constants import GEOWEAVER_DEFAULT_ENDPOINT_URL + + if not id: + raise RuntimeError("Workflow id is missing") + download_geoweaver_jar() + url = f"{GEOWEAVER_DEFAULT_ENDPOINT_URL}/web/detail" + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + form_data = {'type': type, 'id': id} + d = requests.post(url=url, data=form_data, headers=headers) + d = d.json() + d['nodes'] = json.loads(d['nodes']) + try: + from IPython import get_ipython + if 'IPKernelApp' in get_ipython().config: + table_html = create_table([d]) + table_output = widgets.Output() + with table_output: + display(HTML(table_html)) + display(table_output) + except: + return pd.DataFrame([d]) diff --git a/test/test_detail.py b/test/test_detail.py index dc9bd25..5b6d222 100644 --- a/test/test_detail.py +++ b/test/test_detail.py @@ -1,29 +1,55 @@ -from io import StringIO -import sys +import json import unittest -from pygeoweaver.sc_detail import detail_host, detail_process, detail_workflow -from pygeoweaver.utils import get_logger +import pandas as pd +from unittest.mock import patch, Mock +from pygeoweaver import get_logger logger = get_logger(__name__) -def test_detail_process(capfd): - detail_process("not_existing_id") - output, err = capfd.readouterr() - logger.debug("stdout_output" + output) - assert "No process found with id: not_existing_id" in output +class TestDetailFunctions(unittest.TestCase): + @patch('pygeoweaver.utils.download_geoweaver_jar') + @patch('requests.post') + def test_detail_workflow(self, mock_post, mock_download): + from pygeoweaver.sc_detail import detail_workflow + with self.assertRaises(RuntimeError): + detail_workflow(None) + mock_response = Mock() + mock_response.json.return_value = {'nodes': json.dumps({"some_key": "some_value"})} + mock_post.return_value = mock_response + result = detail_workflow('some_id') + self.assertIsInstance(result, pd.DataFrame) -def test_detail_workflow(capfd): - detail_workflow("not_existing_id") - output, err = capfd.readouterr() - logger.debug("stdout_output" + output) - assert "No workflow found with id: not_existing_id" in output + @patch('pygeoweaver.utils.download_geoweaver_jar') + @patch('requests.post') + def test_detail_process(self, mock_post, mock_download): + from pygeoweaver.sc_detail import detail_process + with self.assertRaises(RuntimeError): + detail_process(None) + mock_response = Mock() + mock_response.json.return_value = {'nodes': json.dumps({"some_key": "some_value"})} + mock_post.return_value = mock_response + result = detail_process('some_id') + self.assertIsInstance(result, pd.DataFrame) -def test_detail_host(capfd): - detail_host("not_existing_id") - output, err = capfd.readouterr() - logger.debug("stdout_output" + output) - assert "No host found with id: not_existing_id" in output + @patch('pygeoweaver.utils.download_geoweaver_jar') + @patch('requests.post') + def test_detail_host(self, mock_post, mock_download): + from pygeoweaver.sc_detail import detail_host + + with self.assertRaises(RuntimeError): + detail_host(None) + + mock_response = Mock() + mock_response.json.return_value = {"some_key": "some_value"} + mock_post.return_value = mock_response + + result = detail_host('some_id') + self.assertIsInstance(result, pd.DataFrame) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_import.py b/test/test_import.py new file mode 100644 index 0000000..87aad21 --- /dev/null +++ b/test/test_import.py @@ -0,0 +1,35 @@ +import os +import unittest +from unittest.mock import patch, Mock +from pygeoweaver.sc_import import import_workflow + + +class TestImportWorkflow(unittest.TestCase): + + @patch('os.path.basename') + @patch('os.path.splitext') + @patch('builtins.open') + @patch('requests.post') + def test_import_workflow(self, mock_post, mock_open, mock_splitext, mock_basename): + # Setup + mock_basename.return_value = 'test_file.zip' + mock_splitext.return_value = ('test_file', '.zip') + mock_open.return_value = Mock() + + mock_response = Mock() + mock_response.json.return_value = {'status': 'success'} + mock_post.return_value = mock_response + + # Mocking GEOWEAVER_DEFAULT_ENDPOINT_URL + with patch('pygeoweaver.constants.GEOWEAVER_DEFAULT_ENDPOINT_URL', 'http://example.com'): + # Run + result = import_workflow('/path/to/test_file.zip') + + # Asserts + self.assertEqual(result, 'Import success.') + mock_post.assert_called() + mock_open.assert_called_with('/path/to/test_file.zip', 'rb') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_list.py b/test/test_list.py new file mode 100644 index 0000000..9cd2dd9 --- /dev/null +++ b/test/test_list.py @@ -0,0 +1,56 @@ +import unittest +from unittest.mock import patch, MagicMock +import pygeoweaver # replace 'your_module' with the actual module name where pygeoweaver is defined + + +class TestListHosts(unittest.TestCase): + @patch('IPython.core.display_functions.display') + @patch('requests.post') + def test_list_hosts(self, mock_post, mock_display): + mock_post.return_value.json.return_value = [{'some': 'data'}] + pygeoweaver.create_table = MagicMock(return_value=('table_html', 'some_other_value')) + + with patch('builtins.globals', return_value={'get_ipython': None}): + pygeoweaver.list_hosts() + + +class TestListProcesses(unittest.TestCase): + + @patch('IPython.core.display_functions.display') + @patch('requests.post') + def test_list_processes(self, mock_post, mock_display): + mock_post.return_value.json.return_value = [{'some': 'data'}] + pygeoweaver.create_table = MagicMock(return_value=('table_html', None)) + + with patch('builtins.globals', return_value={'get_ipython': None}): + pygeoweaver.list_processes() + + +class TestListProcessesInWorkflow(unittest.TestCase): + + @patch('json.loads') + @patch('requests.post') + def test_list_processes_in_workflow(self, mock_post, mock_json_loads): + mock_post.return_value.json.return_value = {'nodes': '[{"title": "some_title", "id": "some_id"}]'} + mock_json_loads.return_value = [{'title': 'some_title', 'id': 'some_id'}] + + result = pygeoweaver.list_processes_in_workflow('some_workflow_id') + + expected_result = [{'title': 'some_title', 'id': 'some_id'}] + self.assertEqual(result, expected_result) + + +class TestListWorkflows(unittest.TestCase): + + @patch('IPython.core.display_functions.display') + @patch('requests.post') + def test_list_workflows(self, mock_post, mock_display): + mock_post.return_value.json.return_value = [{'some': 'data'}] + pygeoweaver.create_table = MagicMock(return_value=('table_html', None)) + + with patch('builtins.globals', return_value={'get_ipython': None}): + pygeoweaver.list_workflows() + + +if __name__ == '__main__': + unittest.main()