Merge branch 'master' into boto3

IM/SSH.py

@@ -131,8 +131,15 @@ def __init__(self, host, user, passwd=None, private_key=None, port=22, proxy_hos
 
             self.private_key = private_key
             private_key_obj.seek(0)
-            self.private_key_obj = paramiko.RSAKey.from_private_key(
-                private_key_obj)
+
+            if "BEGIN RSA PRIVATE KEY" in private_key:
+                self.private_key_obj = paramiko.RSAKey.from_private_key(private_key_obj)
+            elif "BEGIN DSA PRIVATE KEY" in private_key:
+                self.private_key_obj = paramiko.DSSKey.from_private_key(private_key_obj)
+            elif "BEGIN EC PRIVATE KEY" in private_key:
+                self.private_key_obj = paramiko.ECDSAKey.from_private_key(private_key_obj)
+            elif "BEGIN OPENSSH PRIVATE KEY" in private_key:
+                self.private_key_obj = paramiko.Ed25519Key.from_private_key(private_key_obj)
 
     def __del__(self):
         self.close()

IM/connectors/Kubernetes.py

@@ -48,7 +48,7 @@ class KubernetesCloudConnector(CloudConnector):
     """Dictionary with a map with the Kubernetes POD states to the IM states."""
 
     def create_request(self, method, url, auth_data, headers=None, body=None):
-        auth_header, _ = self.get_auth_header(auth_data)
+        auth_header, _, _ = self.get_auth_header(auth_data)
         if auth_header:
             if headers is None:
                 headers = {}
@@ -91,7 +91,11 @@ def get_auth_header(self, auth_data):
         if 'namespace' in auth:
             namespace = auth['namespace']
 
-        return auth_header, namespace
+        apps_dns = None
+        if 'apps_dns' in auth:
+            apps_dns = auth['apps_dns']
+
+        return auth_header, namespace, apps_dns
 
     def concrete_system(self, radl_system, str_url, auth_data):
         url = urlparse(str_url)
@@ -329,9 +333,10 @@ def _generate_service_data(self, namespace, name, outports, public):
 
         return service_data
 
-    def create_ingress(self, namespace, name, dns, port, auth_data):
+    def create_ingress(self, namespace, name, dns, port, auth_data, vm):
         try:
-            ingress_data = self._generate_ingress_data(namespace, name, dns, port)
+            _, _, apps_dns = self.get_auth_header(auth_data)
+            ingress_data = self._generate_ingress_data(namespace, name, dns, port, apps_dns, vm)
             self.log_debug("Creating Ingress: %s/%s" % (namespace, name))
             headers = {'Content-Type': 'application/json'}
             uri = "/apis/networking.k8s.io/v1/namespaces/%s/ingresses" % namespace
@@ -347,7 +352,7 @@ def create_ingress(self, namespace, name, dns, port, auth_data):
             self.log_exception("Error creating ingress.")
             return False
 
-    def _generate_ingress_data(self, namespace, name, dns, port):
+    def _generate_ingress_data(self, namespace, name, dns, port, apps_dns, vm):
         ingress_data = self._gen_basic_k8s_elem(namespace, name, 'Ingress', 'networking.k8s.io/v1')
 
         host = None
@@ -364,16 +369,24 @@ def _generate_ingress_data(self, namespace, name, dns, port):
                 secure = True
             if dns_url[1]:
                 host = dns_url[1]
+                if apps_dns and not host.endswith(apps_dns):
+                    if not host.endswith(".") and not apps_dns.startswith("."):
+                        host += "."
+                    host += apps_dns
             if dns_url[2]:
                 path = dns_url[2]
 
+            vm.info.systems[0].setValue('net_interface.0.dns_name', '%s://%s%s' % (dns_url[0], host, path))
+
         ingress_data["metadata"]["annotations"] = {
             "haproxy.router.openshift.io/ip_whitelist": "0.0.0.0/0",
         }
         # Add Let's Encrypt annotation asuming that the cluster has
         # cert-manager installed and the issuer is letsencrypt-prod
         if secure:
             ingress_data["metadata"]["annotations"]["cert-manager.io/cluster-issuer"] = "letsencrypt-prod"
+            ingress_data["metadata"]["annotations"]["route.openshift.io/termination"] = "edge"
+            ingress_data["metadata"]["annotations"]["haproxy.router.openshift.io/redirect-to-https"] = "True"
 
         ingress_data["spec"] = {
             "rules": [
@@ -396,7 +409,7 @@ def _generate_ingress_data(self, namespace, name, dns, port):
         if host:
             ingress_data["spec"]["rules"][0]["host"] = host
 
-        if secure and host:
+        if secure and host and not apps_dns:
             ingress_data["spec"]["tls"] = [{"hosts": [host], "secretName": name + "-tls"}]
 
         return ingress_data
@@ -415,12 +428,13 @@ def _gen_basic_k8s_elem(namespace, name, kind, version="v1"):
     def _get_env_variables(radl_system):
         env_vars = []
         if radl_system.getValue('environment.variables'):
-            keypairs = radl_system.getValue('environment.variables').split(",")
-            for keypair in keypairs:
-                parts = keypair.split("=")
-                key = parts[0].strip()
-                value = parts[1].strip()
-                env_vars.append({'name': key, 'value': value})
+            # Parse the environment variables
+            # The pattern is: key="value" or key=value
+            # in case of value with commas it should be enclosed in double quotes
+            pattern = r'([^,=]+)=(".*?(?<!\\)"|[^,]*)'
+            keypairs = re.findall(pattern, radl_system.getValue('environment.variables'))
+            for key, value in keypairs:
+                env_vars.append({'name': key.strip(), 'value': value.strip(' "')})
         return env_vars
 
     def _generate_pod_data(self, namespace, name, outports, system, volumes, configmaps, tags):
@@ -443,7 +457,7 @@ def _generate_pod_data(self, namespace, name, outports, system, volumes, configm
         if tags:
             for k, v in tags.items():
                 # Remove special characters
-                pod_data['metadata']['labels'][k] = re.sub('[!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~]', '', v).lstrip("_-")
+                pod_data['metadata']['labels'][k] = re.sub('[!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~ ]', '', v).lstrip("_-")
 
         containers = [{
             'name': name,
@@ -503,7 +517,7 @@ def _generate_pod_data(self, namespace, name, outports, system, volumes, configm
         return pod_data
 
     def _get_namespace(self, inf, auth_data):
-        _, namespace = self.get_auth_header(auth_data)
+        _, namespace, _ = self.get_auth_header(auth_data)
         # If the namespace is set in the auth_data use it
         if not namespace:
             # If not by default use the Inf ID as namespace
@@ -553,7 +567,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
             vm = VirtualMachine(inf, None, self.cloud, radl, requested_radl, self)
             vm.destroy = True
             inf.add_vm(vm)
-            pod_name = re.sub('[!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~_]', '-', system.name)
+            pod_name = re.sub('[!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~_ ]', '-', system.name)
 
             volumes = self._create_volumes(namespace, system, pod_name, auth_data)
 
@@ -599,7 +613,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
 
                 if dns_name and outports:
                     port = outports[0].get_local_port()
-                    ingress_created = self.create_ingress(namespace, pod_name, dns_name, port, auth_data)
+                    ingress_created = self.create_ingress(namespace, pod_name, dns_name, port, auth_data, vm)
                     if not ingress_created:
                         vm.info.systems[0].delValue("net_interface.0.dns_name")
 

IM/connectors/OpenStack.py

@@ -1036,6 +1036,14 @@ def delete_networks(self, driver, inf):
                                                                                        router.name,
                                                                                        get_ex_error(ex))
 
+                    for port in driver.ex_list_ports():
+                        if port.extra.get('network_id') == ost_net.id:
+                            try:
+                                self.log_debug("Deleting port %s." % port.id)
+                                port.delete()
+                            except Exception:
+                                self.log_exception("Error deleting port %s." % port.id)
+
                     self.log_info("Deleting net %s." % ost_net.name)
                     driver.ex_delete_network(ost_net)
 

IM/tosca/Tosca.py

@@ -2150,13 +2150,13 @@ def _gen_k8s_volumes(self, node, nodetemplates, value, cont=1):
             cont += 1
         return volumes
 
-    def _gen_k8s_configmaps(self, res, cms):
+    def _gen_k8s_configmaps(self, res, cms, node):
         """Get the configmaps attached to an K8s container."""
         cont = 1
         for cm in cms:
             mount_path = cm.get("deploy_path")
             cm_file = cm.get("file")
-            content = cm.get("properties", {}).get("content", "")
+            content = self._final_function_result(cm.get("properties", {}).get("content", ""), node)
             if content:
                 res.setValue('disk.%d.content' % cont, content)
             # if content is not empty file is ignored
@@ -2186,7 +2186,7 @@ def _gen_k8s_system(self, node, nodetemplates):
         if not image:
             raise Exception("No image specified for K8s container.")
 
-        cont = self._gen_k8s_configmaps(res, cms)
+        cont = self._gen_k8s_configmaps(res, cms, node)
 
         repo = artifact.get("repository", None)
         if repo:
@@ -2206,6 +2206,8 @@ def _gen_k8s_system(self, node, nodetemplates):
                     for k, v in value.items():
                         if variables != "":
                             variables += ","
+                        if ',' in v:
+                            v = '"%s"' % v
                         variables += "%s=%s" % (k, v)
                     res.setValue("environment.variables", variables)
                 elif prop.name == "command":

doc/source/client.rst

@@ -367,6 +367,9 @@ The available keys are:
 
 * ``namespace`` indicates a namespace name to be associated to the Kubernetes credential (from version 1.7.1).
 
+* ``apps_dns`` indicates a DNS domain used by the Kubernetes provider to expose application URLs.
+  (from version 1.7.1).
+
 Vault Credentials support
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -501,6 +504,9 @@ There are several ways to get the EGI AAI token:
 
    token = command(oidc-token OIDC_ACCOUNT)
 
+* It is also possible to get the token using the EGI AAI endpoint. The token can be obtained in the
+   `Check-in Token Portal <https://aai.egi.eu/token/>`_.
+
 * Another way is using the IM-Dashboard (:ref:`use-dashboard`). In the "Advanced" menu, the "Settings"
   item enables getting the some configuration settings as the OIDC issuer or the current user's
   access token.

test/files/privatekeyec.pem

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACAJRqoIjiHNnr8AugARO9zIoSQP+lIRmSkl92mcc8T9JAAAAKANznLhDc5y
+4QAAAAtzc2gtZWQyNTUxOQAAACAJRqoIjiHNnr8AugARO9zIoSQP+lIRmSkl92mcc8T9JA
+AAAECBnGuXE7SFHDP32PbNFbfXkkZNpeHJKG5luU1H7kH1HQlGqgiOIc2evwC6ABE73Mih
+JA/6UhGZKSX3aZxzxP0kAAAAF21pY2FmZXJAREVTS1RPUC02Vk9DNEMzAQIDBAUG
+-----END OPENSSH PRIVATE KEY-----

test/files/tosca_k8s.yml

@@ -56,9 +56,12 @@ topology_template:
             # when the content is not provided, the file is downloaded from the URL
             # otherwise, the file is ignored
             # If the content is base64 encoded, it is assumed to be a K8s Secret
-            content: |
-              [im]
-              REST_API = True
+            content:
+              concat:
+                - |-
+                    [im]
+                    REST_API = 
+                - "True"
         my_secret:
           deploy_path: /etc/secret
           type: tosca.artifacts.File
@@ -86,6 +89,7 @@ topology_template:
         environment:
           MYSQL_ROOT_PASSWORD: { get_input: mysql_root_password }
           MYSQL_DATABASE: "im-db"
+          TEST: "some,value"
       requirements:
         - host: mysql_runtime
       artifacts:

test/unit/AppDB.py

@@ -99,8 +99,6 @@ def test_get_site_url(self, requests):
         requests.side_effect = self.get_response
         res = AppDB.get_site_url("8016G0", "openstack")
         self.assertEqual(res, "https://cloud.recas.ba.infn.it:5000")
-        res = AppDB.get_site_url("8015G0", "occi")
-        self.assertEqual(res, "http://cloud.recas.ba.infn.it:8787/occi/")
 
     @patch('requests.request')
     def test_get_image_id(self, requests):

test/unit/SSH.py

@@ -47,7 +47,7 @@ def test_str(self):
 
     @patch('paramiko.SSHClient')
     def test_test_connectivity(self, ssh_client):
-        ssh = SSHRetry("host", "user", "passwd", read_file_as_string("../files/privatekey.pem"))
+        ssh = SSHRetry("host", "user", "passwd", read_file_as_string("../files/privatekeyec.pem"))
         success = ssh.test_connectivity(5)
         self.assertTrue(success)
 

test/unit/Tosca.py

@@ -442,7 +442,8 @@ def test_tosca_k8s(self):
         self.assertEqual(node.getValue("memory.size"), 1000000000)
         self.assertEqual(node.getValue("disk.1.size"), 10000000000)
         self.assertEqual(node.getValue("disk.1.mount_path"), '/var/lib/mysql')
-        self.assertEqual(node.getValue("environment.variables"), 'MYSQL_ROOT_PASSWORD=my-secret,MYSQL_DATABASE=im-db')
+        self.assertEqual(node.getValue("environment.variables"),
+                         'MYSQL_ROOT_PASSWORD=my-secret,MYSQL_DATABASE=im-db,TEST="some,value"')
         self.assertEqual(node.getValue("net_interface.0.connection"), 'mysql_container_priv')
         self.assertIsNone(node.getValue("net_interface.1.connection"))
         net = radl.get_network_by_id('mysql_container_priv')
@@ -457,7 +458,7 @@ def test_tosca_k8s(self):
         net = radl.get_network_by_id('im_container_pub')
         self.assertEqual(net.getValue("outports"), '30880/tcp-8800/tcp')
         self.assertEqual(net.getValue("outbound"), 'yes')
-        self.assertEqual(node.getValue("disk.1.content"), '[im]\nREST_API = True\n')
+        self.assertEqual(node.getValue("disk.1.content"), '[im]\nREST_API = True')
         self.assertEqual(node.getValue("disk.1.mount_path"), '/etc/im/im.cfg')
         self.assertEqual(node.getValue("disk.2.content"), 'c29tZSBlbmNvZGVkIGNvbnRlbnQ=')
         self.assertEqual(node.getValue("disk.2.mount_path"), '/etc/secret')

test/unit/connectors/Kubernetes.py

@@ -145,7 +145,7 @@ def test_20_launch(self, save_data, requests):
             memory.size>=512m and
             net_interface.0.connection = 'net' and
             net_interface.0.dns_name = 'https://ingress.domain.com/path' and
-            environment.variables = 'var=some_val' and
+            environment.variables = 'var=some_val,var2="some,val2"' and
             instance_tags = 'key=_inva:lid_' and
             disk.0.os.name = 'linux' and
             disk.0.image.url = 'docker://someimage' and
@@ -240,7 +240,8 @@ def test_20_launch(self, save_data, requests):
                             "limits": {"cpu": "1", "memory": "536870912"},
                             "requests": {"cpu": "1", "memory": "536870912"},
                         },
-                        "env": [{"name": "var", "value": "some_val"}],
+                        "env": [{"name": "var", "value": "some_val"},
+                                {"name": "var2", "value": "some,val2"}],
                         "volumeMounts": [{"name": "test-1", "mountPath": "/mnt"},
                                          {'mountPath': '/etc/config', 'name': 'test-cm-2',
                                           'readOnly': True, 'subPath': 'config'},
@@ -297,7 +298,9 @@ def test_20_launch(self, save_data, requests):
                 "namespace": "somenamespace",
                 "annotations": {
                     "cert-manager.io/cluster-issuer": "letsencrypt-prod",
-                    "haproxy.router.openshift.io/ip_whitelist": "0.0.0.0/0"
+                    "haproxy.router.openshift.io/ip_whitelist": "0.0.0.0/0",
+                    "haproxy.router.openshift.io/redirect-to-https": "True",
+                    "route.openshift.io/termination": "edge"
                 },
             },
             "spec": {

test/unit/connectors/OpenStack.py

@@ -801,6 +801,7 @@ def test_60_finalize(self, sleep, get_driver):
         driver.ex_list_security_groups.return_value = [sg1, sg2, sg3]
 
         net1 = MagicMock()
+        net1.id = 'net1id'
         net1.name = "im-infid-private"
         net1.cidr = None
         net1.extra = {'subnets': ["subnet1"]}
@@ -822,6 +823,12 @@ def test_60_finalize(self, sleep, get_driver):
         driver.detach_volume.return_value = True
         driver.ex_remove_security_group_from_node.return_value = True
 
+        port = MagicMock()
+        port.id = "port1"
+        port.extra = {'network_id': net1.id}
+        port.delete.return_value = True
+        driver.ex_list_ports.return_value = [port]
+
         vm.volumes = ['volid']
         vm.dns_entries = [('dydns:secret@test', 'domain.com.', '8.8.8.8')]
         success, _ = ost_cloud.finalize(vm, True, auth)
@@ -842,6 +849,7 @@ def test_60_finalize(self, sleep, get_driver):
         self.assertEqual(fip.delete.call_args_list, [call()])
         self.assertEqual(fip2.delete.call_count, 0)
         self.assertEqual(driver.ex_detach_floating_ip_from_node.call_args_list[0][0], (node, fip))
+        self.assertEqual(port.delete.call_args_list, [call()])
 
         vm.floating_ips = ['158.42.1.1']
         success, _ = ost_cloud.finalize(vm, True, auth)