From 0651857722166f557901719badd77abfea2f2113 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 15 Apr 2021 12:46:05 +0200 Subject: [PATCH 1/6] Add dns deletion functionality --- app/__init__.py | 3 ++ app/utils.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 75 insertions(+) diff --git a/app/__init__.py b/app/__init__.py index dbc8894e8..e8f3bed9f 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -786,6 +786,9 @@ def manage_inf(infid=None, op=None): force = False if 'force' in form_data and form_data['force'] != "0": force = True + success, msg = utils.delete_dns_record(infid, im, auth_data) + if not success: + app.logger.error('Error deleting DNS record: %s', (msg)) response = im.delete_inf(infid, force, auth_data) if not response.ok: raise Exception(response.text) diff --git a/app/utils.py b/app/utils.py index 5b783aa54..b4a78dfc8 100644 --- a/app/utils.py +++ b/app/utils.py @@ -289,3 +289,74 @@ def exchange_token_with_audience(oidc_url, client_id, client_secret, oidc_token, deserialized_oidc_response = json.loads(oidc_response.text) return deserialized_oidc_response['access_token'] + +def delete_dns_record(infid, im, auth_data): + """Helper function to delete DNS registry created with the tosca.nodes.ec3.DNSRegistry node type""" + template = "" + try: + response = im.get_inf_property(infid, 'tosca', auth_data) + if not response.ok: + raise Exception(response.text) + template = response.text + except Exception as ex: + return False, "Error getting infrastructure template: %s" % ex + + try: + yaml_template = yaml.safe_load(template) + for node in list(yaml_template['topology_template']['node_templates'].values()): + if node["type"] == "tosca.nodes.ec3.DNSRegistry": + record = node["properties"]["record_name"] + domain = node["properties"]["domain_name"] + credentials = node["properties"]["dns_service_credentials"]["token"] + delete_route53_record(record, domain, credentials) + except Exception as ex: + return False, "Error deleting DNS record: %s" % ex + + return True, "" + +def delete_route53_record(record, domain, credentials): + import boto3 + + if credentials.startswith("arn:aws:iam"): + sts_client = boto3.client('sts') + + assumed_role_object=sts_client.assume_role( + RoleArn="arn:aws:iam::account-of-role-to-assume:role/name-of-role", + RoleSessionName="AssumeRoleSession1" + ) + + credentials=assumed_role_object['Credentials'] + + route53=boto3.client( + 'route53', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + ) + elif credentials.find(":") != -1: + parts = credentials.split(":") + route53=boto3.client( + 'route53', + aws_access_key_id=parts[0], + aws_secret_access_key=parts[1] + ) + else: + raise Exception("Invalid credentials") + + zone = route53.list_hosted_zones_by_name(DNSName=domain)["HostedZones"][0] + + record = route53.list_resource_record_sets(HostedZoneId=zone['Id'], + StartRecordType='A', + StartRecordName='%s.%s.' % (record, domain), + MaxItems='1')["ResourceRecordSets"][0] + + route53.change_resource_record_sets( + HostedZoneId=zone['Id'], + ChangeBatch={ + 'Changes': [ + { + 'Action': 'DELETE', + 'ResourceRecordSet': record + } + ] + }) diff --git a/requirements.txt b/requirements.txt index a9557badc..961328dbe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ cryptography==3.3.1 apscheduler==3.7.0 Flask-APScheduler==1.12.0 Flask-WTF==0.14.3 +boto3==1.17.52 From beb2eaa9fdaf79a60eaa46b00f13fc29016d6e17 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 15 Apr 2021 16:45:15 +0200 Subject: [PATCH 2/6] Add generate dns name randomly --- app/__init__.py | 13 ++ app/utils.py | 364 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 373 insertions(+), 4 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index e8f3bed9f..82d64bb09 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -528,6 +528,15 @@ def add_auth_to_template(template, auth_data): return template + def add_record_name_to_template(template): + # Add a random name in the DNS record name + + for node in list(template['topology_template']['node_templates'].values()): + if node["type"] == "tosca.nodes.ec3.DNSRegistry": + node["properties"]["record_name"] = utils.generate_random_name() + + return template + def set_inputs_to_template(template, inputs): # Add the image to all compute nodes @@ -599,6 +608,9 @@ def createdep(): template = add_auth_to_template(template, auth_data) + # Specially added for OSCAR clusters + template = add_record_name_to_template(template) + inputs = {k: v for (k, v) in form_data.items() if not k.startswith("extra_opts.")} app.logger.debug("Parameters: " + json.dumps(inputs)) @@ -786,6 +798,7 @@ def manage_inf(infid=None, op=None): force = False if 'force' in form_data and form_data['force'] != "0": force = True + # Specially added for OSCAR clusters success, msg = utils.delete_dns_record(infid, im, auth_data) if not success: app.logger.error('Error deleting DNS record: %s', (msg)) diff --git a/app/utils.py b/app/utils.py index b4a78dfc8..a2b0b241e 100644 --- a/app/utils.py +++ b/app/utils.py @@ -32,6 +32,7 @@ from app import appdb from fnmatch import fnmatch from hashlib import md5 +from random import randint from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) @@ -307,7 +308,7 @@ def delete_dns_record(infid, im, auth_data): if node["type"] == "tosca.nodes.ec3.DNSRegistry": record = node["properties"]["record_name"] domain = node["properties"]["domain_name"] - credentials = node["properties"]["dns_service_credentials"]["token"] + credentials = node["properties"]["dns_service_credentials"]["token"].strip() delete_route53_record(record, domain, credentials) except Exception as ex: return False, "Error deleting DNS record: %s" % ex @@ -315,6 +316,9 @@ def delete_dns_record(infid, im, auth_data): return True, "" def delete_route53_record(record, domain, credentials): + if not (record and domain and credentials): + return + import boto3 if credentials.startswith("arn:aws:iam"): @@ -346,9 +350,9 @@ def delete_route53_record(record, domain, credentials): zone = route53.list_hosted_zones_by_name(DNSName=domain)["HostedZones"][0] record = route53.list_resource_record_sets(HostedZoneId=zone['Id'], - StartRecordType='A', - StartRecordName='%s.%s.' % (record, domain), - MaxItems='1')["ResourceRecordSets"][0] + StartRecordType='A', + StartRecordName='%s.%s.' % (record, domain), + MaxItems='1')["ResourceRecordSets"][0] route53.change_resource_record_sets( HostedZoneId=zone['Id'], @@ -360,3 +364,355 @@ def delete_route53_record(record, domain, credentials): } ] }) + +def generate_random_name(): + left = [ + "admiring", + "adoring", + "affectionate", + "agitated", + "amazing", + "angry", + "awesome", + "beautiful", + "blissful", + "bold", + "boring", + "brave", + "busy", + "charming", + "clever", + "cool", + "compassionate", + "competent", + "condescending", + "confident", + "cranky", + "crazy", + "dazzling", + "determined", + "distracted", + "dreamy", + "eager", + "ecstatic", + "elastic", + "elated", + "elegant", + "eloquent", + "epic", + "exciting", + "fervent", + "festive", + "flamboyant", + "focused", + "friendly", + "frosty", + "funny", + "gallant", + "gifted", + "goofy", + "gracious", + "great", + "happy", + "hardcore", + "heuristic", + "hopeful", + "hungry", + "infallible", + "inspiring", + "interesting", + "intelligent", + "jolly", + "jovial", + "keen", + "kind", + "laughing", + "loving", + "lucid", + "magical", + "mystifying", + "modest", + "musing", + "naughty", + "nervous", + "nice", + "nifty", + "nostalgic", + "objective", + "optimistic", + "peaceful", + "pedantic", + "pensive", + "practical", + "priceless", + "quirky", + "quizzical", + "recursing", + "relaxed", + "reverent", + "romantic", + "sad", + "serene", + "sharp", + "silly", + "sleepy", + "stoic", + "strange", + "stupefied", + "suspicious", + "sweet", + "tender", + "thirsty", + "trusting", + "unruffled", + "upbeat", + "vibrant", + "vigilant", + "vigorous", + "wizardly", + "wonderful", + "xenodochial", + "youthful", + "zealous", + "zen" + ] + rigth = [ + "albattani", + "allen", + "almeida", + "antonelli", + "agnesi", + "archimedes", + "ardinghelli", + "aryabhata", + "austin", + "babbage", + "banach", + "banzai", + "bardeen", + "bartik", + "bassi", + "beaver", + "bell", + "benz", + "bhabha", + "bhaskara", + "black", + "blackburn", + "blackwell", + "bohr", + "booth", + "borg", + "bose", + "bouman", + "boyd", + "brahmagupta", + "brattain", + "brown", + "buck", + "burnell", + "cannon", + "carson", + "cartwright", + "carver", + "cerf", + "chandrasekhar", + "chaplygin", + "chatelet", + "chatterjee", + "chebyshev", + "cohen", + "chaum", + "clarke", + "colden", + "cori", + "cray", + "curran", + "curie", + "darwin", + "davinci", + "dewdney", + "dhawan", + "diffie", + "dijkstra", + "dirac", + "driscoll", + "dubinsky", + "easley", + "edison", + "einstein", + "elbakyan", + "elgamal", + "elion", + "ellis", + "engelbart", + "euclid", + "euler", + "faraday", + "feistel", + "fermat", + "fermi", + "feynman", + "franklin", + "gagarin", + "galileo", + "galois", + "ganguly", + "gates", + "gauss", + "germain", + "goldberg", + "goldstine", + "goldwasser", + "golick", + "goodall", + "gould", + "greider", + "grothendieck", + "haibt", + "hamilton", + "haslett", + "hawking", + "hellman", + "heisenberg", + "hermann", + "herschel", + "hertz", + "heyrovsky", + "hodgkin", + "hofstadter", + "hoover", + "hopper", + "hugle", + "hypatia", + "ishizaka", + "jackson", + "jang", + "jemison", + "jennings", + "jepsen", + "johnson", + "joliot", + "jones", + "kalam", + "kapitsa", + "kare", + "keldysh", + "keller", + "kepler", + "khayyam", + "khorana", + "kilby", + "kirch", + "knuth", + "kowalevski", + "lalande", + "lamarr", + "lamport", + "leakey", + "leavitt", + "lederberg", + "lehmann", + "lewin", + "lichterman", + "liskov", + "lovelace", + "lumiere", + "mahavira", + "margulis", + "matsumoto", + "maxwell", + "mayer", + "mccarthy", + "mcclintock", + "mclaren", + "mclean", + "mcnulty", + "mendel", + "mendeleev", + "meitner", + "meninsky", + "merkle", + "mestorf", + "mirzakhani", + "montalcini", + "moore", + "morse", + "murdock", + "moser", + "napier", + "nash", + "neumann", + "newton", + "nightingale", + "nobel", + "noether", + "northcutt", + "noyce", + "panini", + "pare", + "pascal", + "pasteur", + "payne", + "perlman", + "pike", + "poincare", + "poitras", + "proskuriakova", + "ptolemy", + "raman", + "ramanujan", + "ride", + "ritchie", + "rhodes", + "robinson", + "roentgen", + "rosalind", + "rubin", + "saha", + "sammet", + "sanderson", + "satoshi", + "shamir", + "shannon", + "shaw", + "shirley", + "shockley", + "shtern", + "sinoussi", + "snyder", + "solomon", + "spence", + "stonebraker", + "sutherland", + "swanson", + "swartz", + "swirles", + "taussig", + "tereshkova", + "tesla", + "tharp", + "thompson", + "torvalds", + "tu", + "turing", + "varahamihira", + "vaughan", + "visvesvaraya", + "volhard", + "villani", + "wescoff", + "wilbur", + "wiles", + "williams", + "williamson", + "wilson", + "wing", + "wozniak", + "wright", + "wu", + "yalow", + "yonath", + "zhukovsky" + ] + return "%s-%s%d" % (left[randint(0, len(left))], rigth[randint(0, len(rigth))], randint(0, 9)) From 137db862b649d14b0bb342e97af3a833d333deba Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 21 Apr 2021 13:07:57 +0200 Subject: [PATCH 3/6] Fix error showing IPs --- app/__init__.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 82d64bb09..bf92efe52 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -238,16 +238,17 @@ def showvminfo(): del vminfo["disk.0.os.name"] cont = 0 - while "net_interface.%s.ip" % cont in vminfo: - if cont > 0: - nets += Markup('
') - nets += Markup('') - nets += Markup(' %s' % cont) - nets += ": %s" % vminfo["net_interface.%s.ip" % cont] - del vminfo["net_interface.%s.ip" % cont] - if "net_interface.%s.dns_name" % cont in vminfo: - nets += " (%s)" % vminfo["net_interface.%s.dns_name" % cont] - del vminfo["net_interface.%s.dns_name" % cont] + while "net_interface.%s.connection" % cont in vminfo: + if "net_interface.%s.ip" % cont in vminfo: + if cont > 0: + nets += Markup('
') + nets += Markup('') + nets += Markup(' %s' % cont) + nets += ": %s" % vminfo["net_interface.%s.ip" % cont] + del vminfo["net_interface.%s.ip" % cont] + if "net_interface.%s.dns_name" % cont in vminfo: + nets += " (%s)" % vminfo["net_interface.%s.dns_name" % cont] + del vminfo["net_interface.%s.dns_name" % cont] cont += 1 From 5c7c601d9bde61b0fbf871ae69fc9f6235bf040d Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 21 Apr 2021 13:08:54 +0200 Subject: [PATCH 4/6] Fix error showing IPs --- app/__init__.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 82d64bb09..bf92efe52 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -238,16 +238,17 @@ def showvminfo(): del vminfo["disk.0.os.name"] cont = 0 - while "net_interface.%s.ip" % cont in vminfo: - if cont > 0: - nets += Markup('
') - nets += Markup('') - nets += Markup(' %s' % cont) - nets += ": %s" % vminfo["net_interface.%s.ip" % cont] - del vminfo["net_interface.%s.ip" % cont] - if "net_interface.%s.dns_name" % cont in vminfo: - nets += " (%s)" % vminfo["net_interface.%s.dns_name" % cont] - del vminfo["net_interface.%s.dns_name" % cont] + while "net_interface.%s.connection" % cont in vminfo: + if "net_interface.%s.ip" % cont in vminfo: + if cont > 0: + nets += Markup('
') + nets += Markup('') + nets += Markup(' %s' % cont) + nets += ": %s" % vminfo["net_interface.%s.ip" % cont] + del vminfo["net_interface.%s.ip" % cont] + if "net_interface.%s.dns_name" % cont in vminfo: + nets += " (%s)" % vminfo["net_interface.%s.dns_name" % cont] + del vminfo["net_interface.%s.dns_name" % cont] cont += 1 From ddc701ef6f9c0fca714f2a0f05fdbce2e6a4245c Mon Sep 17 00:00:00 2001 From: micafer Date: Fri, 23 Apr 2021 12:40:17 +0200 Subject: [PATCH 5/6] Show outports --- app/__init__.py | 20 ++++++++++++++++++-- app/templates/vminfo.html | 10 ++++++++++ app/utils.py | 11 +++++++++++ requirements.txt | 2 +- 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index dbc8894e8..7a980b5ca 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -218,7 +218,9 @@ def showvminfo(): flash("Error retrieving VM info: \n" + response.text, 'error') else: app.logger.debug("VM Info: %s" % response.text) - vminfo = utils.format_json_radl(response.json()["radl"]) + radl_json = response.json()["radl"] + outports = utils.get_out_ports(radl_json) + vminfo = utils.format_json_radl(radl_json) if "cpu.arch" in vminfo: del vminfo["cpu.arch"] if "state" in vminfo: @@ -279,7 +281,21 @@ def showvminfo(): cont += 1 - return render_template('vminfo.html', infid=infid, vmid=vmid, vminfo=vminfo, + str_outports = "" + for port in outports: + str_outports += Markup(' %s' % port.get_remote_port()) + if not port.is_range(): + if port.get_remote_port() != port.get_local_port(): + str_outports += Markup(' ' + ' %s' % + port.get_local_port()) + else: + str_outports += Markup(' : %s' % + port.get_local_port()) + str_outports += Markup('
') + + return render_template('vminfo.html', infid=infid, vmid=vmid, vminfo=vminfo, outports=str_outports, state=state, nets=nets, deployment=deployment, disks=disks) @app.route('/managevm///', methods=['POST']) diff --git a/app/templates/vminfo.html b/app/templates/vminfo.html index 0ecf95cac..9e477d34c 100644 --- a/app/templates/vminfo.html +++ b/app/templates/vminfo.html @@ -86,6 +86,16 @@

VM ID {{ vmid }}:

{{ nets }} + {% if outports %} + + + Ports: + + + {{ outports }} + + + {% endif %} HW Features: diff --git a/app/utils.py b/app/utils.py index 5b783aa54..1ad28ed02 100644 --- a/app/utils.py +++ b/app/utils.py @@ -28,6 +28,7 @@ import ast import time import sys +from radl.radl import outport from flask import flash, g from app import appdb from fnmatch import fnmatch @@ -196,6 +197,16 @@ def format_json_radl(vminfo): return res +def get_out_ports(vminfo): + outports = [] + for elem in vminfo: + # Search outports of public nets + if elem["class"] == "network" and "outports" in elem and "outbound" in elem and elem["outbound"] == "yes": + outports = outport.parseOutPorts(elem["outports"]) + + return outports + + def to_pretty_json(value): return json.dumps(value, sort_keys=True, indent=4, separators=(',', ': ')) diff --git a/requirements.txt b/requirements.txt index a9557badc..8b25b677f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ requests==2.25.1 PyYAML==5.4.1 packaging==20.9 xmltodict==0.12.0 -radl==1.1.3 +radl==1.1.5 cryptography==3.3.1 apscheduler==3.7.0 Flask-APScheduler==1.12.0 From 14adee7324c692f0a2802ac74d9c75e602b60b3b Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 26 Apr 2021 11:02:01 +0200 Subject: [PATCH 6/6] Fix style --- app/utils.py | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/app/utils.py b/app/utils.py index a2b0b241e..f21f7923c 100644 --- a/app/utils.py +++ b/app/utils.py @@ -291,8 +291,9 @@ def exchange_token_with_audience(oidc_url, client_id, client_secret, oidc_token, return deserialized_oidc_response['access_token'] + def delete_dns_record(infid, im, auth_data): - """Helper function to delete DNS registry created with the tosca.nodes.ec3.DNSRegistry node type""" + """Helper function to delete DNS registry created with the tosca.nodes.ec3.DNSRegistry node type.""" template = "" try: response = im.get_inf_property(infid, 'tosca', auth_data) @@ -312,34 +313,19 @@ def delete_dns_record(infid, im, auth_data): delete_route53_record(record, domain, credentials) except Exception as ex: return False, "Error deleting DNS record: %s" % ex - + return True, "" + def delete_route53_record(record, domain, credentials): if not (record and domain and credentials): - return + return import boto3 - if credentials.startswith("arn:aws:iam"): - sts_client = boto3.client('sts') - - assumed_role_object=sts_client.assume_role( - RoleArn="arn:aws:iam::account-of-role-to-assume:role/name-of-role", - RoleSessionName="AssumeRoleSession1" - ) - - credentials=assumed_role_object['Credentials'] - - route53=boto3.client( - 'route53', - aws_access_key_id=credentials['AccessKeyId'], - aws_secret_access_key=credentials['SecretAccessKey'], - aws_session_token=credentials['SessionToken'], - ) - elif credentials.find(":") != -1: + if credentials.find(":") != -1: parts = credentials.split(":") - route53=boto3.client( + route53 = boto3.client( 'route53', aws_access_key_id=parts[0], aws_secret_access_key=parts[1] @@ -350,9 +336,9 @@ def delete_route53_record(record, domain, credentials): zone = route53.list_hosted_zones_by_name(DNSName=domain)["HostedZones"][0] record = route53.list_resource_record_sets(HostedZoneId=zone['Id'], - StartRecordType='A', - StartRecordName='%s.%s.' % (record, domain), - MaxItems='1')["ResourceRecordSets"][0] + StartRecordType='A', + StartRecordName='%s.%s.' % (record, domain), + MaxItems='1')["ResourceRecordSets"][0] route53.change_resource_record_sets( HostedZoneId=zone['Id'], @@ -365,6 +351,7 @@ def delete_route53_record(record, domain, credentials): ] }) + def generate_random_name(): left = [ "admiring",