From 774c4f79a4627db7d55692e19cf3b2ef68db843d Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 23 Sep 2024 12:37:27 +0200 Subject: [PATCH 01/15] Add Check-in Token Portal in docs --- doc/source/client.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/client.rst b/doc/source/client.rst index 128c3245..08ee77f8 100644 --- a/doc/source/client.rst +++ b/doc/source/client.rst @@ -501,6 +501,9 @@ There are several ways to get the EGI AAI token: token = command(oidc-token OIDC_ACCOUNT) +* It is also possible to get the token using the EGI AAI endpoint. The token can be obtained in the + `Check-in Token Portal `_. + * Another way is using the IM-Dashboard (:ref:`use-dashboard`). In the "Advanced" menu, the "Settings" item enables getting the some configuration settings as the OIDC issuer or the current user's access token. From 330562a93b6fc15ea5f0cb327fbf1320b050e0f1 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 26 Sep 2024 16:45:21 +0200 Subject: [PATCH 02/15] Implements #1611 in OST --- IM/connectors/OpenStack.py | 101 +++++++++++++++++++++++------- test/unit/connectors/OpenStack.py | 46 ++++++++++++++ 2 files changed, 126 insertions(+), 21 deletions(-) diff --git a/IM/connectors/OpenStack.py b/IM/connectors/OpenStack.py index 1b58635c..c3d38fb9 100644 --- a/IM/connectors/OpenStack.py +++ b/IM/connectors/OpenStack.py @@ -1653,6 +1653,24 @@ def _get_security_group(self, driver, sg_name): self.log_exception("Error getting security groups.") return None + def add_security_group_rules(self, driver, outports, sg): + """Add the security group rules to the security group""" + for outport in outports: + if outport.is_range(): + to_port = outport.get_port_end() + from_port = outport.get_port_init() + else: + to_port = from_port = outport.get_remote_port() + + try: + driver.ex_create_security_group_rule(sg, outport.get_protocol(), + from_port, to_port, + outport.get_remote_cidr()) + except Exception as ex: + self.log_warn("Exception adding SG rules: %s" % get_ex_error(ex)) + self.error_messages += ("Exception adding port range: %s-%s to SG rules.\n" % + (from_port, to_port)) + def create_security_groups(self, driver, inf, radl): res = [] system = radl.systems[0] @@ -1709,27 +1727,7 @@ def create_security_groups(self, driver, inf, radl): if network.isPublic() or network.getValue("proxy_host"): outports = self.add_ssh_port(outports) - for outport in outports: - if outport.is_range(): - try: - driver.ex_create_security_group_rule(sg, outport.get_protocol(), - outport.get_port_init(), - outport.get_port_end(), - outport.get_remote_cidr()) - except Exception as ex: - self.log_warn("Exception adding SG rules: %s" % get_ex_error(ex)) - self.error_messages += ("Exception adding port range: %s-%s to SG rules.\n" % - (outport.get_port_init(), outport.get_port_end())) - else: - try: - driver.ex_create_security_group_rule(sg, outport.get_protocol(), - outport.get_remote_port(), - outport.get_remote_port(), - outport.get_remote_cidr()) - except Exception as ex: - self.log_warn("Exception adding SG rules: %s" % get_ex_error(ex)) - self.error_messages += ("Exception adding port %s to SG rules.\n" % - outport.get_remote_port()) + self.add_security_group_rules(driver, outports, sg) return res @@ -1958,6 +1956,67 @@ def alterVM(self, vm, radl, auth_data): if not success: return (success, msg) + success, msg = self.alter_security_groups(vm, radl, auth_data) + if not success: + return (success, msg) + + return (True, "") + + def alter_security_groups(self, vm, radl, auth_data): + driver = self.get_driver(auth_data) + + # First check if the node is "routed" + for network in radl.networks: + if network.getValue('router'): + self.log_info("Network has a router set. Skip SG rules modification.") + return (True, "") + + for network in radl.networks: + outports = network.getOutPorts() or [] + + old_network = vm.info.get_network_by_id(network.id) + old_outports = [] + if old_network: + old_outports = old_network.getOutPorts() or [] + + if old_outports == outports: + self.log_debug("No changes in the SG rules for network %s." % network.id) + break + + # open always SSH port on public nets or private with proxy host + if old_network and old_network.isPublic() or old_network.getValue("proxy_host"): + old_outports = self.add_ssh_port(old_outports) + if network.isPublic() or network.getValue("proxy_host"): + outports = self.add_ssh_port(outports) + + sg_name = network.getValue("sg_name") + if not sg_name: + sg_name = "im-%s-%s" % (str(vm.inf.id), network.id) + + sg = self._get_security_group(driver, sg_name) + if not sg: + self.log_error("Error updating security group: %s. It does not exist." % sg_name) + break + + # Delete old SG rules + for rule in sg.rules: + # For each rule in the SG, check if it is in the old_outports and remove it + for outport in old_outports: + protocol = outport.get_protocol() + if outport.is_range(): + to_port = outport.get_port_end() + from_port = outport.get_port_init() + else: + to_port = from_port = outport.get_remote_port() + if rule.from_port == from_port and rule.to_port == to_port and rule.ip_protocol == protocol: + try: + driver.ex_delete_security_group_rule(rule) + except Exception as ex: + self.log_warn("Exception removing old SG rules: %s" % get_ex_error(ex)) + + # Add new SG rules + self.add_security_group_rules(driver, outports, sg) + return (True, "") def resizeVM(self, vm, radl, auth_data): diff --git a/test/unit/connectors/OpenStack.py b/test/unit/connectors/OpenStack.py index 4995cf60..f74cca4c 100644 --- a/test/unit/connectors/OpenStack.py +++ b/test/unit/connectors/OpenStack.py @@ -636,6 +636,7 @@ def test_55_alter(self, add_elastic_ip_from_pool, get_driver): ost_cloud = self.get_ost_cloud() inf = MagicMock() + inf.id = "infid" vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud, 1) vm.volumes = [] @@ -737,6 +738,51 @@ def test_55_alter(self, add_elastic_ip_from_pool, get_driver): self.assertEqual(driver.ex_detach_floating_ip_from_node.call_args_list[0][0], (node, fip)) self.assertIsNone(vm.requested_radl.systems[0].getValue('net_interface.0.ip')) + radl_data = """ + network net (outbound = 'yes' and outports = '8080') + system test ( + cpu.arch='x86_64' and + cpu.count=1 and + memory.size=512m and + net_interface.0.connection = 'net' and + net_interface.0.ip = '8.8.8.8' and + net_interface.0.dns_name = 'test' and + disk.0.os.name = 'linux' and + disk.0.image.url = 'one://server.com/1' and + disk.0.os.credentials.username = 'user' and + disk.0.os.credentials.password = 'pass' + )""" + radl = radl_parse.parse_radl(radl_data) + vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud, 1) + + new_radl_data = """ + network net (outbound = 'yes' and outports = '8081') + system test ( + net_interface.0.connection = 'net' + )""" + new_radl = radl_parse.parse_radl(new_radl_data) + + sg = MagicMock() + sg.name = "im-infid-net" + rule = MagicMock() + rule.id = "rid" + rule.from_port = 8080 + rule.to_port = 8080 + rule.ip_protocol = 'tcp' + sg.rules = [rule] + driver.ex_list_security_groups.return_value = [sg] + driver.ex_delete_security_group_rule.return_value = True + driver.ex_create_security_group_rule.return_value = True + + success, _ = ost_cloud.alterVM(vm, new_radl, auth) + self.assertTrue(success, msg="ERROR: modifying VM info.") + self.assertEqual(driver.ex_delete_security_group_rule.call_count, 1) + self.assertEqual(driver.ex_delete_security_group_rule.call_args_list[0][0], (rule,)) + self.assertEqual(driver.ex_create_security_group_rule.call_count, 2) + self.assertEqual(driver.ex_create_security_group_rule.call_args_list[0][0], + (sg, 'tcp', 8081, 8081, '0.0.0.0/0')) + self.assertEqual(driver.ex_create_security_group_rule.call_args_list[1][0], + (sg, 'tcp', 22, 22, '0.0.0.0/0')) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver') From 4af918af5c59f60e007bac58906543343b8a42e9 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 27 Sep 2024 09:28:25 +0200 Subject: [PATCH 03/15] Dettach port from nets before deleting --- IM/connectors/OpenStack.py | 8 ++++++++ test/unit/connectors/OpenStack.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/IM/connectors/OpenStack.py b/IM/connectors/OpenStack.py index c847e935..d5810ee0 100644 --- a/IM/connectors/OpenStack.py +++ b/IM/connectors/OpenStack.py @@ -1019,6 +1019,14 @@ def delete_networks(self, driver, inf): router.name, get_ex_error(ex)) + for port in driver.ex_list_ports(): + if port.extra.get('network_id') == ost_net.id: + try: + self.log_debug("Deleting port %s." % port.id) + port.delete() + except Exception: + self.log_exception("Error deleting port %s." % port.id) + self.log_info("Deleting net %s." % ost_net.name) driver.ex_delete_network(ost_net) diff --git a/test/unit/connectors/OpenStack.py b/test/unit/connectors/OpenStack.py index 692fc761..390fe611 100644 --- a/test/unit/connectors/OpenStack.py +++ b/test/unit/connectors/OpenStack.py @@ -801,6 +801,7 @@ def test_60_finalize(self, sleep, get_driver): driver.ex_list_security_groups.return_value = [sg1, sg2, sg3] net1 = MagicMock() + net1.id = 'net1id' net1.name = "im-infid-private" net1.cidr = None net1.extra = {'subnets': ["subnet1"]} @@ -822,6 +823,12 @@ def test_60_finalize(self, sleep, get_driver): driver.detach_volume.return_value = True driver.ex_remove_security_group_from_node.return_value = True + port = MagicMock() + port.id = "port1" + port.extra = {'network_id': net1.id} + port.delete.return_value = True + driver.ex_list_ports.return_value = [port] + vm.volumes = ['volid'] vm.dns_entries = [('dydns:secret@test', 'domain.com.', '8.8.8.8')] success, _ = ost_cloud.finalize(vm, True, auth) @@ -842,6 +849,7 @@ def test_60_finalize(self, sleep, get_driver): self.assertEqual(fip.delete.call_args_list, [call()]) self.assertEqual(fip2.delete.call_count, 0) self.assertEqual(driver.ex_detach_floating_ip_from_node.call_args_list[0][0], (node, fip)) + self.assertEqual(port.delete.call_args_list, [call()]) vm.floating_ips = ['158.42.1.1'] success, _ = ost_cloud.finalize(vm, True, auth) From 9bca00c96b52e10c4505fd9c6bfd264dfa5f9eef Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 8 Oct 2024 15:52:09 +0200 Subject: [PATCH 04/15] Remove spaces in pod labels --- IM/connectors/Kubernetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 1aad6871..9cf87fbc 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -443,7 +443,7 @@ def _generate_pod_data(self, namespace, name, outports, system, volumes, configm if tags: for k, v in tags.items(): # Remove special characters - pod_data['metadata']['labels'][k] = re.sub('[!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~]', '', v).lstrip("_-") + pod_data['metadata']['labels'][k] = re.sub('[!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~ ]', '', v).lstrip("_-") containers = [{ 'name': name, From 566df5116be4b8f2c0cb6de9a50d286ad8835047 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 10 Oct 2024 08:32:42 +0200 Subject: [PATCH 05/15] Implements #1613 --- IM/SSH.py | 11 +++++++++-- test/files/privatekeyec.pem | 7 +++++++ test/unit/SSH.py | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 test/files/privatekeyec.pem diff --git a/IM/SSH.py b/IM/SSH.py index 3976948e..966793fc 100644 --- a/IM/SSH.py +++ b/IM/SSH.py @@ -131,8 +131,15 @@ def __init__(self, host, user, passwd=None, private_key=None, port=22, proxy_hos self.private_key = private_key private_key_obj.seek(0) - self.private_key_obj = paramiko.RSAKey.from_private_key( - private_key_obj) + + if "BEGIN RSA PRIVATE KEY" in private_key: + self.private_key_obj = paramiko.RSAKey.from_private_key(private_key_obj) + elif "BEGIN DSA PRIVATE KEY" in private_key: + self.private_key_obj = paramiko.DSSKey.from_private_key(private_key_obj) + elif "BEGIN EC PRIVATE KEY" in private_key: + self.private_key_obj = paramiko.ECDSAKey.from_private_key(private_key_obj) + elif "BEGIN OPENSSH PRIVATE KEY" in private_key: + self.private_key_obj = paramiko.Ed25519Key.from_private_key(private_key_obj) def __del__(self): self.close() diff --git a/test/files/privatekeyec.pem b/test/files/privatekeyec.pem new file mode 100644 index 00000000..7880f8cd --- /dev/null +++ b/test/files/privatekeyec.pem @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACAJRqoIjiHNnr8AugARO9zIoSQP+lIRmSkl92mcc8T9JAAAAKANznLhDc5y +4QAAAAtzc2gtZWQyNTUxOQAAACAJRqoIjiHNnr8AugARO9zIoSQP+lIRmSkl92mcc8T9JA +AAAECBnGuXE7SFHDP32PbNFbfXkkZNpeHJKG5luU1H7kH1HQlGqgiOIc2evwC6ABE73Mih +JA/6UhGZKSX3aZxzxP0kAAAAF21pY2FmZXJAREVTS1RPUC02Vk9DNEMzAQIDBAUG +-----END OPENSSH PRIVATE KEY----- diff --git a/test/unit/SSH.py b/test/unit/SSH.py index 162bf4d1..a7c18349 100644 --- a/test/unit/SSH.py +++ b/test/unit/SSH.py @@ -47,7 +47,7 @@ def test_str(self): @patch('paramiko.SSHClient') def test_test_connectivity(self, ssh_client): - ssh = SSHRetry("host", "user", "passwd", read_file_as_string("../files/privatekey.pem")) + ssh = SSHRetry("host", "user", "passwd", read_file_as_string("../files/privatekeyec.pem")) success = ssh.test_connectivity(5) self.assertTrue(success) From c9875cb93743f190b8775dc45fcd963595d7f46c Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 10 Oct 2024 08:37:38 +0200 Subject: [PATCH 06/15] Revert "Implements #1611 in OST" This reverts commit 330562a93b6fc15ea5f0cb327fbf1320b050e0f1. --- IM/connectors/OpenStack.py | 101 +++++++----------------------- test/unit/connectors/OpenStack.py | 46 -------------- 2 files changed, 21 insertions(+), 126 deletions(-) diff --git a/IM/connectors/OpenStack.py b/IM/connectors/OpenStack.py index c3d38fb9..1b58635c 100644 --- a/IM/connectors/OpenStack.py +++ b/IM/connectors/OpenStack.py @@ -1653,24 +1653,6 @@ def _get_security_group(self, driver, sg_name): self.log_exception("Error getting security groups.") return None - def add_security_group_rules(self, driver, outports, sg): - """Add the security group rules to the security group""" - for outport in outports: - if outport.is_range(): - to_port = outport.get_port_end() - from_port = outport.get_port_init() - else: - to_port = from_port = outport.get_remote_port() - - try: - driver.ex_create_security_group_rule(sg, outport.get_protocol(), - from_port, to_port, - outport.get_remote_cidr()) - except Exception as ex: - self.log_warn("Exception adding SG rules: %s" % get_ex_error(ex)) - self.error_messages += ("Exception adding port range: %s-%s to SG rules.\n" % - (from_port, to_port)) - def create_security_groups(self, driver, inf, radl): res = [] system = radl.systems[0] @@ -1727,7 +1709,27 @@ def create_security_groups(self, driver, inf, radl): if network.isPublic() or network.getValue("proxy_host"): outports = self.add_ssh_port(outports) - self.add_security_group_rules(driver, outports, sg) + for outport in outports: + if outport.is_range(): + try: + driver.ex_create_security_group_rule(sg, outport.get_protocol(), + outport.get_port_init(), + outport.get_port_end(), + outport.get_remote_cidr()) + except Exception as ex: + self.log_warn("Exception adding SG rules: %s" % get_ex_error(ex)) + self.error_messages += ("Exception adding port range: %s-%s to SG rules.\n" % + (outport.get_port_init(), outport.get_port_end())) + else: + try: + driver.ex_create_security_group_rule(sg, outport.get_protocol(), + outport.get_remote_port(), + outport.get_remote_port(), + outport.get_remote_cidr()) + except Exception as ex: + self.log_warn("Exception adding SG rules: %s" % get_ex_error(ex)) + self.error_messages += ("Exception adding port %s to SG rules.\n" % + outport.get_remote_port()) return res @@ -1956,67 +1958,6 @@ def alterVM(self, vm, radl, auth_data): if not success: return (success, msg) - success, msg = self.alter_security_groups(vm, radl, auth_data) - if not success: - return (success, msg) - - return (True, "") - - def alter_security_groups(self, vm, radl, auth_data): - driver = self.get_driver(auth_data) - - # First check if the node is "routed" - for network in radl.networks: - if network.getValue('router'): - self.log_info("Network has a router set. Skip SG rules modification.") - return (True, "") - - for network in radl.networks: - outports = network.getOutPorts() or [] - - old_network = vm.info.get_network_by_id(network.id) - old_outports = [] - if old_network: - old_outports = old_network.getOutPorts() or [] - - if old_outports == outports: - self.log_debug("No changes in the SG rules for network %s." % network.id) - break - - # open always SSH port on public nets or private with proxy host - if old_network and old_network.isPublic() or old_network.getValue("proxy_host"): - old_outports = self.add_ssh_port(old_outports) - if network.isPublic() or network.getValue("proxy_host"): - outports = self.add_ssh_port(outports) - - sg_name = network.getValue("sg_name") - if not sg_name: - sg_name = "im-%s-%s" % (str(vm.inf.id), network.id) - - sg = self._get_security_group(driver, sg_name) - if not sg: - self.log_error("Error updating security group: %s. It does not exist." % sg_name) - break - - # Delete old SG rules - for rule in sg.rules: - # For each rule in the SG, check if it is in the old_outports and remove it - for outport in old_outports: - protocol = outport.get_protocol() - if outport.is_range(): - to_port = outport.get_port_end() - from_port = outport.get_port_init() - else: - to_port = from_port = outport.get_remote_port() - if rule.from_port == from_port and rule.to_port == to_port and rule.ip_protocol == protocol: - try: - driver.ex_delete_security_group_rule(rule) - except Exception as ex: - self.log_warn("Exception removing old SG rules: %s" % get_ex_error(ex)) - - # Add new SG rules - self.add_security_group_rules(driver, outports, sg) - return (True, "") def resizeVM(self, vm, radl, auth_data): diff --git a/test/unit/connectors/OpenStack.py b/test/unit/connectors/OpenStack.py index f74cca4c..4995cf60 100644 --- a/test/unit/connectors/OpenStack.py +++ b/test/unit/connectors/OpenStack.py @@ -636,7 +636,6 @@ def test_55_alter(self, add_elastic_ip_from_pool, get_driver): ost_cloud = self.get_ost_cloud() inf = MagicMock() - inf.id = "infid" vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud, 1) vm.volumes = [] @@ -738,51 +737,6 @@ def test_55_alter(self, add_elastic_ip_from_pool, get_driver): self.assertEqual(driver.ex_detach_floating_ip_from_node.call_args_list[0][0], (node, fip)) self.assertIsNone(vm.requested_radl.systems[0].getValue('net_interface.0.ip')) - radl_data = """ - network net (outbound = 'yes' and outports = '8080') - system test ( - cpu.arch='x86_64' and - cpu.count=1 and - memory.size=512m and - net_interface.0.connection = 'net' and - net_interface.0.ip = '8.8.8.8' and - net_interface.0.dns_name = 'test' and - disk.0.os.name = 'linux' and - disk.0.image.url = 'one://server.com/1' and - disk.0.os.credentials.username = 'user' and - disk.0.os.credentials.password = 'pass' - )""" - radl = radl_parse.parse_radl(radl_data) - vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud, 1) - - new_radl_data = """ - network net (outbound = 'yes' and outports = '8081') - system test ( - net_interface.0.connection = 'net' - )""" - new_radl = radl_parse.parse_radl(new_radl_data) - - sg = MagicMock() - sg.name = "im-infid-net" - rule = MagicMock() - rule.id = "rid" - rule.from_port = 8080 - rule.to_port = 8080 - rule.ip_protocol = 'tcp' - sg.rules = [rule] - driver.ex_list_security_groups.return_value = [sg] - driver.ex_delete_security_group_rule.return_value = True - driver.ex_create_security_group_rule.return_value = True - - success, _ = ost_cloud.alterVM(vm, new_radl, auth) - self.assertTrue(success, msg="ERROR: modifying VM info.") - self.assertEqual(driver.ex_delete_security_group_rule.call_count, 1) - self.assertEqual(driver.ex_delete_security_group_rule.call_args_list[0][0], (rule,)) - self.assertEqual(driver.ex_create_security_group_rule.call_count, 2) - self.assertEqual(driver.ex_create_security_group_rule.call_args_list[0][0], - (sg, 'tcp', 8081, 8081, '0.0.0.0/0')) - self.assertEqual(driver.ex_create_security_group_rule.call_args_list[1][0], - (sg, 'tcp', 22, 22, '0.0.0.0/0')) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver') From 10695d4a58213c50dc790a236986e7009e13d4ce Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 10 Oct 2024 08:39:08 +0200 Subject: [PATCH 07/15] remove spaces from pod names --- IM/connectors/Kubernetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 9cf87fbc..dd9be68f 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -553,7 +553,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): vm = VirtualMachine(inf, None, self.cloud, radl, requested_radl, self) vm.destroy = True inf.add_vm(vm) - pod_name = re.sub('[!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~_]', '-', system.name) + pod_name = re.sub('[!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~_ ]', '-', system.name) volumes = self._create_volumes(namespace, system, pod_name, auth_data) From b7d3b946cb2b51d0a34d7d550982d110bce4921e Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 10 Oct 2024 08:49:14 +0200 Subject: [PATCH 08/15] remove occi test --- test/unit/AppDB.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/unit/AppDB.py b/test/unit/AppDB.py index d716ebb7..f407e4d1 100644 --- a/test/unit/AppDB.py +++ b/test/unit/AppDB.py @@ -99,8 +99,6 @@ def test_get_site_url(self, requests): requests.side_effect = self.get_response res = AppDB.get_site_url("8016G0", "openstack") self.assertEqual(res, "https://cloud.recas.ba.infn.it:5000") - res = AppDB.get_site_url("8015G0", "occi") - self.assertEqual(res, "http://cloud.recas.ba.infn.it:8787/occi/") @patch('requests.request') def test_get_image_id(self, requests): From c037bc8346ebbf1b352e281242e45cb213f2cc1a Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 16 Oct 2024 12:37:27 +0200 Subject: [PATCH 09/15] Add apps_dns field --- IM/connectors/Kubernetes.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 1aad6871..e3d01831 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -48,7 +48,7 @@ class KubernetesCloudConnector(CloudConnector): """Dictionary with a map with the Kubernetes POD states to the IM states.""" def create_request(self, method, url, auth_data, headers=None, body=None): - auth_header, _ = self.get_auth_header(auth_data) + auth_header, _, _ = self.get_auth_header(auth_data) if auth_header: if headers is None: headers = {} @@ -91,7 +91,11 @@ def get_auth_header(self, auth_data): if 'namespace' in auth: namespace = auth['namespace'] - return auth_header, namespace + apps_dns = None + if 'apps_dns' in auth: + apps_dns = auth['apps_dns'] + + return auth_header, namespace, apps_dns def concrete_system(self, radl_system, str_url, auth_data): url = urlparse(str_url) @@ -331,7 +335,8 @@ def _generate_service_data(self, namespace, name, outports, public): def create_ingress(self, namespace, name, dns, port, auth_data): try: - ingress_data = self._generate_ingress_data(namespace, name, dns, port) + _, _, apps_dns = self.get_auth_header(auth_data) + ingress_data = self._generate_ingress_data(namespace, name, dns, port, apps_dns) self.log_debug("Creating Ingress: %s/%s" % (namespace, name)) headers = {'Content-Type': 'application/json'} uri = "/apis/networking.k8s.io/v1/namespaces/%s/ingresses" % namespace @@ -347,7 +352,7 @@ def create_ingress(self, namespace, name, dns, port, auth_data): self.log_exception("Error creating ingress.") return False - def _generate_ingress_data(self, namespace, name, dns, port): + def _generate_ingress_data(self, namespace, name, dns, port, apps_dns): ingress_data = self._gen_basic_k8s_elem(namespace, name, 'Ingress', 'networking.k8s.io/v1') host = None @@ -364,6 +369,8 @@ def _generate_ingress_data(self, namespace, name, dns, port): secure = True if dns_url[1]: host = dns_url[1] + if apps_dns and not host.endswith(apps_dns): + host += "." + apps_dns if dns_url[2]: path = dns_url[2] @@ -503,7 +510,7 @@ def _generate_pod_data(self, namespace, name, outports, system, volumes, configm return pod_data def _get_namespace(self, inf, auth_data): - _, namespace = self.get_auth_header(auth_data) + _, namespace, _ = self.get_auth_header(auth_data) # If the namespace is set in the auth_data use it if not namespace: # If not by default use the Inf ID as namespace From e58098e10abee29a1669fc54fd35dd666ee3b4cc Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 17 Oct 2024 10:04:56 +0200 Subject: [PATCH 10/15] Minor change --- IM/connectors/Kubernetes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index c915bc24..f0eef49d 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -370,7 +370,9 @@ def _generate_ingress_data(self, namespace, name, dns, port, apps_dns): if dns_url[1]: host = dns_url[1] if apps_dns and not host.endswith(apps_dns): - host += "." + apps_dns + if not host.endswith(".") and not apps_dns.startswith("."): + host += "." + host += apps_dns if dns_url[2]: path = dns_url[2] From 3a1e267d7c32c84ae0ce2e83da86ad6862b40b0a Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 17 Oct 2024 10:16:40 +0200 Subject: [PATCH 11/15] Update docs --- doc/source/client.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/client.rst b/doc/source/client.rst index 08ee77f8..2e3297eb 100644 --- a/doc/source/client.rst +++ b/doc/source/client.rst @@ -367,6 +367,9 @@ The available keys are: * ``namespace`` indicates a namespace name to be associated to the Kubernetes credential (from version 1.7.1). +* ``apps_dns`` indicates a DNS domain used by the Kubernetes provider to expose application URLs. + (from version 1.7.1). + Vault Credentials support ^^^^^^^^^^^^^^^^^^^^^^^^^ From f99b8f092f88a6adfe9f1369e51dfcefe102682e Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 17 Oct 2024 15:53:28 +0200 Subject: [PATCH 12/15] Accept commas in env variables --- IM/connectors/Kubernetes.py | 13 +++++++------ IM/tosca/Tosca.py | 2 ++ test/files/tosca_k8s.yml | 1 + test/unit/Tosca.py | 3 ++- test/unit/connectors/Kubernetes.py | 5 +++-- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index f0eef49d..b4653795 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -424,12 +424,13 @@ def _gen_basic_k8s_elem(namespace, name, kind, version="v1"): def _get_env_variables(radl_system): env_vars = [] if radl_system.getValue('environment.variables'): - keypairs = radl_system.getValue('environment.variables').split(",") - for keypair in keypairs: - parts = keypair.split("=") - key = parts[0].strip() - value = parts[1].strip() - env_vars.append({'name': key, 'value': value}) + # Parse the environment variables + # The pattern is: key="value" or key=value + # in case of value with commas it should be enclosed in double quotes + pattern = r'([^,=]+)=(".*?(?=512m and net_interface.0.connection = 'net' and net_interface.0.dns_name = 'https://ingress.domain.com/path' and - environment.variables = 'var=some_val' and + environment.variables = 'var=some_val,var2="some,val2"' and instance_tags = 'key=_inva:lid_' and disk.0.os.name = 'linux' and disk.0.image.url = 'docker://someimage' and @@ -240,7 +240,8 @@ def test_20_launch(self, save_data, requests): "limits": {"cpu": "1", "memory": "536870912"}, "requests": {"cpu": "1", "memory": "536870912"}, }, - "env": [{"name": "var", "value": "some_val"}], + "env": [{"name": "var", "value": "some_val"}, + {"name": "var2", "value": "some,val2"}], "volumeMounts": [{"name": "test-1", "mountPath": "/mnt"}, {'mountPath': '/etc/config', 'name': 'test-cm-2', 'readOnly': True, 'subPath': 'config'}, From e2de8a18f84ca370468713eda599aaa71e2295f1 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 18 Oct 2024 09:34:20 +0200 Subject: [PATCH 13/15] Enable use functions in CMs --- IM/tosca/Tosca.py | 6 +++--- test/files/tosca_k8s.yml | 9 ++++++--- test/unit/Tosca.py | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index bd767d53..86820239 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -2150,13 +2150,13 @@ def _gen_k8s_volumes(self, node, nodetemplates, value, cont=1): cont += 1 return volumes - def _gen_k8s_configmaps(self, res, cms): + def _gen_k8s_configmaps(self, res, cms, node): """Get the configmaps attached to an K8s container.""" cont = 1 for cm in cms: mount_path = cm.get("deploy_path") cm_file = cm.get("file") - content = cm.get("properties", {}).get("content", "") + content = self._final_function_result(cm.get("properties", {}).get("content", ""), node) if content: res.setValue('disk.%d.content' % cont, content) # if content is not empty file is ignored @@ -2186,7 +2186,7 @@ def _gen_k8s_system(self, node, nodetemplates): if not image: raise Exception("No image specified for K8s container.") - cont = self._gen_k8s_configmaps(res, cms) + cont = self._gen_k8s_configmaps(res, cms, node) repo = artifact.get("repository", None) if repo: diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml index c75534a5..51590d8d 100644 --- a/test/files/tosca_k8s.yml +++ b/test/files/tosca_k8s.yml @@ -56,9 +56,12 @@ topology_template: # when the content is not provided, the file is downloaded from the URL # otherwise, the file is ignored # If the content is base64 encoded, it is assumed to be a K8s Secret - content: | - [im] - REST_API = True + content: + concat: + - |- + [im] + REST_API = + - "True" my_secret: deploy_path: /etc/secret type: tosca.artifacts.File diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index 59fcc375..0325ed0c 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -458,7 +458,7 @@ def test_tosca_k8s(self): net = radl.get_network_by_id('im_container_pub') self.assertEqual(net.getValue("outports"), '30880/tcp-8800/tcp') self.assertEqual(net.getValue("outbound"), 'yes') - self.assertEqual(node.getValue("disk.1.content"), '[im]\nREST_API = True\n') + self.assertEqual(node.getValue("disk.1.content"), '[im]\nREST_API = True') self.assertEqual(node.getValue("disk.1.mount_path"), '/etc/im/im.cfg') self.assertEqual(node.getValue("disk.2.content"), 'c29tZSBlbmNvZGVkIGNvbnRlbnQ=') self.assertEqual(node.getValue("disk.2.mount_path"), '/etc/secret') From 3abb60bb36a812da88a091eb9eecb8a535c94149 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 21 Oct 2024 09:42:52 +0200 Subject: [PATCH 14/15] Fix https support in OKD --- IM/connectors/Kubernetes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index b4653795..e488e0d6 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -383,6 +383,8 @@ def _generate_ingress_data(self, namespace, name, dns, port, apps_dns): # cert-manager installed and the issuer is letsencrypt-prod if secure: ingress_data["metadata"]["annotations"]["cert-manager.io/cluster-issuer"] = "letsencrypt-prod" + ingress_data["metadata"]["annotations"]["route.openshift.io/termination"] = "edge" + ingress_data["metadata"]["annotations"]["haproxy.router.openshift.io/redirect-to-https"] = "True" ingress_data["spec"] = { "rules": [ @@ -405,7 +407,7 @@ def _generate_ingress_data(self, namespace, name, dns, port, apps_dns): if host: ingress_data["spec"]["rules"][0]["host"] = host - if secure and host: + if secure and host and not apps_dns: ingress_data["spec"]["tls"] = [{"hosts": [host], "secretName": name + "-tls"}] return ingress_data From 7fb97b558354739d71f8c7176ebf07e0032b57e0 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 21 Oct 2024 10:16:54 +0200 Subject: [PATCH 15/15] Fix endpoint output --- IM/connectors/Kubernetes.py | 10 ++++++---- test/unit/connectors/Kubernetes.py | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index e488e0d6..8cbb3367 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -333,10 +333,10 @@ def _generate_service_data(self, namespace, name, outports, public): return service_data - def create_ingress(self, namespace, name, dns, port, auth_data): + def create_ingress(self, namespace, name, dns, port, auth_data, vm): try: _, _, apps_dns = self.get_auth_header(auth_data) - ingress_data = self._generate_ingress_data(namespace, name, dns, port, apps_dns) + ingress_data = self._generate_ingress_data(namespace, name, dns, port, apps_dns, vm) self.log_debug("Creating Ingress: %s/%s" % (namespace, name)) headers = {'Content-Type': 'application/json'} uri = "/apis/networking.k8s.io/v1/namespaces/%s/ingresses" % namespace @@ -352,7 +352,7 @@ def create_ingress(self, namespace, name, dns, port, auth_data): self.log_exception("Error creating ingress.") return False - def _generate_ingress_data(self, namespace, name, dns, port, apps_dns): + def _generate_ingress_data(self, namespace, name, dns, port, apps_dns, vm): ingress_data = self._gen_basic_k8s_elem(namespace, name, 'Ingress', 'networking.k8s.io/v1') host = None @@ -376,6 +376,8 @@ def _generate_ingress_data(self, namespace, name, dns, port, apps_dns): if dns_url[2]: path = dns_url[2] + vm.info.systems[0].setValue('net_interface.0.dns_name', '%s://%s%s' % (dns_url[0], host, path)) + ingress_data["metadata"]["annotations"] = { "haproxy.router.openshift.io/ip_whitelist": "0.0.0.0/0", } @@ -611,7 +613,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): if dns_name and outports: port = outports[0].get_local_port() - ingress_created = self.create_ingress(namespace, pod_name, dns_name, port, auth_data) + ingress_created = self.create_ingress(namespace, pod_name, dns_name, port, auth_data, vm) if not ingress_created: vm.info.systems[0].delValue("net_interface.0.dns_name") diff --git a/test/unit/connectors/Kubernetes.py b/test/unit/connectors/Kubernetes.py index 70c75191..3e8667d7 100755 --- a/test/unit/connectors/Kubernetes.py +++ b/test/unit/connectors/Kubernetes.py @@ -298,7 +298,9 @@ def test_20_launch(self, save_data, requests): "namespace": "somenamespace", "annotations": { "cert-manager.io/cluster-issuer": "letsencrypt-prod", - "haproxy.router.openshift.io/ip_whitelist": "0.0.0.0/0" + "haproxy.router.openshift.io/ip_whitelist": "0.0.0.0/0", + "haproxy.router.openshift.io/redirect-to-https": "True", + "route.openshift.io/termination": "edge" }, }, "spec": {