Merge pull request #184 from openvstorage/develop

Promote master

src/api/decorators/flask_requests.py

@@ -21,7 +21,8 @@
 import json
 import time
 import subprocess
-from flask import Response
+from flask import Response, request
+from functools import wraps
 from ovs_extensions.api.exceptions import HttpBadRequestException
 from ovs_extensions.dal.base import ObjectNotFoundException
 
@@ -34,6 +35,40 @@ class HTTPRequestFlaskDecorators(object):
     logger = None
     version = None
 
+    @classmethod
+    def provide_request_data(cls, f):
+        # type: (callable) -> callable
+        """
+        This decorator feeds in the request data in the function with the 'request_data' keyword
+        Used for backwards compatibility (transition to application JSON heading with client/server)
+        - Attempts to read JSON contents
+        :return: The wrapped function
+        :rtype: callable
+        """
+        @wraps(f)
+        def wrap(*args, **kwargs):
+            # type: (*any, **any) -> any
+            """
+            Wrapper function
+            :return: Output of the wrapped function
+            :rtype: any
+            """
+            request_data = request.get_json()
+            if request_data is None:
+                # Try the old route. All keys are potentially JSON serialized within the form (it was a mess)
+                request_data = {}
+                for key, value in request.form.iteritems():
+                    try:
+                        value = json.loads(value)
+                    except ValueError:
+                        # Not a valid JSON, could be string (who can tell at this point...)
+                        pass
+                    request_data[key] = value
+            if 'request_data' in kwargs:
+                raise ValueError('request_data is a reserved argument for the decorator')
+            return f(*args, request_data=request_data, **kwargs)
+        return wrap
+
     @classmethod
     def get(cls, route, authenticate=True):
         """

src/constants/config.py

@@ -0,0 +1,25 @@
+# Copyright (C) 2018 iNuron NV
+#
+# This file is part of Open vStorage Open Source Edition (OSE),
+# as available from
+#
+#      http://www.openvstorage.org and
+#      http://www.openvstorage.com.
+#
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3)
+# as published by the Free Software Foundation, in version 3 as it comes
+# in the LICENSE.txt file of the Open vStorage OSE distribution.
+#
+# Open vStorage is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY of any kind.
+
+"""
+Shared strings
+"""
+
+ARAKOON_NAME = 'cacc'
+ARAKOON_NAME_UNITTEST = 'unittest-cacc'
+CACC_LOCATION = '/opt/OpenvStorage/config/arakoon_cacc.ini'
+CONFIG_STORE_LOCATION = '/opt/OpenvStorage/config/framework.json'
+COMPONENTS_KEY = '/ovs/machines/{0}/components'  # Format will be the machine ID

src/generic/configuration/clients/base_keyvalue.py

@@ -275,10 +275,11 @@ def assert_value(self, key, value, transaction=None):
         :return: None
         :rtype: NoneType
         """
+        key = self._clean_key(key)
         return self._client.assert_value(key, value, transaction=transaction)
 
     def assert_exists(self, key, transaction=None):
-        # type: (str, str) -> None
+        # type: (str, Any) -> None
         """
         Asserts that a key exists
         :param key: Key to assert for
@@ -288,6 +289,7 @@ def assert_exists(self, key, transaction=None):
         :return: None
         :rtype: NoneType
         """
+        key = self._clean_key(key)
         return self._client.assert_exists(key, transaction=transaction)
 
     def begin_transaction(self):

src/generic/configuration/configuration.py

@@ -24,8 +24,10 @@
 import collections
 from random import randint
 from subprocess import check_output
-from ovs_extensions.log.logger import Logger
+from ovs_extensions.constants.config import CACC_LOCATION, COMPONENTS_KEY
+from ovs_extensions.generic.system import System
 from ovs_extensions.packages.packagefactory import PackageFactory
+from ovs_extensions.log.logger import Logger
 # Import for backwards compatibility/easier access
 from ovs_extensions.generic.configuration.exceptions import ConfigurationNotFoundException as NotFoundException
 from ovs_extensions.generic.configuration.exceptions import ConfigurationAssertionException  # New exception, not mapping
@@ -54,7 +56,7 @@ class Configuration(object):
     """
 
     BASE_KEY = '/ovs/framework'
-    CACC_LOCATION = None
+    CACC_LOCATION = CACC_LOCATION
     EDITION_KEY = '{0}/edition'.format(BASE_KEY)
 
     _clients = {}
@@ -120,7 +122,7 @@ def extract_key_from_path(cls, path):
 
     @classmethod
     def get(cls, key, raw=False, **kwargs):
-        # type: (str, bool, **kwargs) -> any
+        # type: (str, bool, **any) -> any
         """
         Get value from the configuration store
         :param key: Key to get
@@ -149,7 +151,7 @@ def get(cls, key, raw=False, **kwargs):
 
     @classmethod
     def _get(cls, key, raw=False, **kwargs):
-        # type: (str, bool, **kwargs) -> Union[dict, None]
+        # type: (str, bool, **any) -> Union[dict, None]
         data = cls._passthrough(method='get',
                                 key=key,
                                 **kwargs)
@@ -159,7 +161,7 @@ def _get(cls, key, raw=False, **kwargs):
 
     @classmethod
     def set(cls, key, value, raw=False, transaction=None):
-        # type: (str, any, raw, str) -> None
+        # type: (str, any, bool, str) -> None
         """
         Set value in the configuration store
         :param key: Key to store
@@ -248,7 +250,7 @@ def delete(cls, key, remove_root=False, raw=False, transaction=None):
 
     @classmethod
     def _delete(cls, key, recursive, transaction=None):
-        # type: (str, bool) -> None
+        # type: (str, bool, str) -> None
         return cls._passthrough(method='delete',
                                 key=key,
                                 recursive=recursive,
@@ -377,7 +379,7 @@ def get_client(cls):
 
     @classmethod
     def _passthrough(cls, method, *args, **kwargs):
-        # type: (str, *args, **kwargs) -> any
+        # type: (str, *any, **any) -> any
         if os.environ.get('RUNNING_UNITTESTS') == 'True':
             store = 'unittest'
         else:
@@ -508,3 +510,60 @@ def safely_store(cls, callback, max_retries=20):
                 time.sleep(randint(0, 25) / 100.0)
                 cls._logger.info('Executing the passed function again')
         return return_value
+
+    @classmethod
+    def register_usage(cls, component_identifier):
+        # type: (str) -> List[str]
+        """
+        Registers that the component is using configuration management
+        When sharing the same configuration management for multiple processes, these registrations can be used to determine
+        if the configuration access can be wiped on the node
+        :param component_identifier: Identifier of the component
+        :type component_identifier: str
+        :return: The currently registered users
+        :rtype: List[str]
+        """
+        registration_key = cls.get_registration_key()
+
+        def _register_user_callback():
+            registered_applications = cls.get(registration_key, default=None)
+            new_registered_applications = (registered_applications or []) + [component_identifier]
+            return [(registration_key, new_registered_applications, registered_applications)]
+        return cls.safely_store(_register_user_callback, 20)[0][1]
+
+    @classmethod
+    def get_registration_key(cls):
+        # type: () -> str
+        """
+        Generate the key to register the component under
+        :return: The registration key
+        :rtype: str
+        """
+        return COMPONENTS_KEY.format(System.get_my_machine_id())
+
+    @classmethod
+    def unregister_usage(cls, component_identifier):
+        # type: (str) -> List[str]
+        """
+        Registers that the component is using configuration management
+        When sharing the same configuration management for multiple processes, these registrations can be used to determine
+        if the configuration access can be wiped on the node
+        :param component_identifier: Identifier of the component
+        :type component_identifier: str
+        :return: The currently registered users
+        :rtype: List[str]
+        """
+        registration_key = cls.get_registration_key()
+
+        def _unregister_user_callback():
+            registered_applications = cls.get(registration_key, default=None)  # type: List[str]
+            if not registered_applications:
+                # No more entries. Save an empty list
+                new_registered_applications = []
+            else:
+                new_registered_applications = registered_applications[:]
+                if component_identifier in registered_applications:
+                    new_registered_applications.remove(component_identifier)
+            return [(registration_key, new_registered_applications, registered_applications)]
+
+        return cls.safely_store(_unregister_user_callback, 20)[0][1]

src/generic/disk.py

@@ -17,6 +17,7 @@
 Disk module
 """
 
+import os
 import re
 import json
 import time
@@ -444,26 +445,82 @@ def retrieve_alias_mapping(cls, ssh_client=None):
         return name_alias_mapping
 
     @classmethod
-    def model_devices(cls, ssh_client=None, name_alias_mapping=None):
-        # type: (Optional[SSHClient], Optional[AliasMapping]) -> Tuple[List[Disk], AliasMapping]
+    def model_devices(cls, ssh_client=None, name_alias_mapping=None, s3=False):
+        # type: (Optional[SSHClient], Optional[AliasMapping], Optional[bool]) -> Tuple[List[Disk], AliasMapping]
         """
         Model all disks that are currently on this machine
         :param ssh_client: SSHClient instance
         :type ssh_client: SSHClient
         :param name_alias_mapping: The name to alias mapping (Optional)
         :type name_alias_mapping: dict
+        :param s3: Whether or not to account for AWS ec2 instances
+        :type s3: bool
         :return: A list of modeled disks, The name to alias mapping used, the alias to name mapping used
         :rtype: Tuple[List[Disk], dict, dict]
         """
         ssh_client = ssh_client or SSHClient('127.0.0.1', username='root')
         if not name_alias_mapping:
             name_alias_mapping = cls.retrieve_alias_mapping(ssh_client)
 
+            if s3:
+                name_alias_mapping.update(cls.map_s3_volumes())
+
         block_devices = cls._model_block_devices(ssh_client)
         cls.logger.info('Starting to iterate over disks')
         disks = cls._model_devices(ssh_client, name_alias_mapping, block_devices)
         return disks, name_alias_mapping
 
+    @classmethod
+    def rename_to_aws(cls, name):
+        # type: (str) -> str
+        """
+        Rename a regular disk to aws disks.
+        Sda -> xvda
+        :param name: name of the disk to be renamed
+        :type name: str
+        :return: new diskname
+        :rtype: str
+        """
+        name = os.path.rsplit(name)[-1]  # Last part of the path is the name of the device
+        if name.startswith('sd'):
+            name = name.replace('sd', 'xvd')
+        return os.path.join('/dev', name)
+
+    @classmethod
+    def convert_to_virtual_id(cls, id):
+        # type: (str) -> str
+        """
+        Add the path mapping to the ID
+        :param id: Volume id to be formatted to path
+        :type id: str
+        :return: /dev/disk/by-virtual-id/<vol-id>
+        """
+        return os.path.join('/dev/disk/by-virtual-id', id)
+
+    @classmethod
+    def map_s3_volumes(cls):
+        # type: () -> Dict[str,str]
+        """
+        Fetch all S3 volumes accessible on the environment
+        :return: All S3 disk names with their mapped volume-IDs
+        """
+        try:
+            from ec2_metadata import ec2_metadata
+            import boto3
+        except ImportError as ex:
+            raise RuntimeError('Failed to load python package: {0}'.format(ex))
+
+        filter = [{'Name': 'attachment.instance-id', 'Values': [ec2_metadata.instance_id]}]
+        ec2 = boto3.resource('ec2', region_name=ec2_metadata.region)
+        volumes = ec2.volumes.filter(Filters=filter)
+        name_map = {}
+        for volume in volumes:
+            for device in volume.attachments:
+                name = cls.rename_to_aws(device['Device'])
+                volume_id = cls.convert_to_virtual_id(device['VolumeId'])
+                name_map[name] = [volume_id]
+        return name_map
+
     @classmethod
     def _model_devices(cls, ssh_client, name_alias_mapping, entries):
         # type: (SSHClient, AliasMapping, List[LSBLKEntry]) -> List[Disk]

src/generic/system.py

@@ -182,3 +182,13 @@ def get_free_ports(cls, selected_range, exclude=None, amount=1, client=None):
         if amount == 0:
             return free_ports
         raise ValueError('Unable to find the requested amount of free ports')
+
+    @staticmethod
+    def get_component_identifier():
+        # type: () -> str
+        """
+        Retrieve the identifier of the component
+        :return: The ID of the component
+        :rtype: str
+        """
+        raise NotImplementedError('The generic implmentation has no record of which component it is used in')

src/generic/tests/test_generic.py

@@ -0,0 +1,40 @@
+# -*- coding: UTF-8 -*-
+#  Copyright (C) 2016 iNuron NV
+#
+# This file is part of Open vStorage Open Source Edition (OSE),
+# as available from
+#
+#      http://www.openvstorage.org and
+#      http://www.openvstorage.com.
+#
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3)
+# as published by the Free Software Foundation, in version 3 as it comes
+# in the LICENSE.txt file of the Open vStorage OSE distribution.
+#
+# Open vStorage is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY of any kind.
+
+"""
+Test module for the Extensions
+"""
+
+import unittest
+from ovs_extensions.generic.toolbox import ExtensionsToolbox
+
+class ExtensionsToolboxTest(unittest.TestCase):
+
+    def test_filter_dict_for_none(self):
+        d = {'a': 'a',
+             'b': {'b1': 'b1',
+                   'b2': None},
+             'c': None,
+             'd': {'d1': {'d11': {'d111': 'd111'}}},
+             'e': {'e1': None}}
+
+        result_dict = {'a': 'a',
+                       'b': {'b1': 'b1'},
+                       'd': {'d1': {'d11': {'d111': 'd111'}}}}
+        filtered_dict = ExtensionsToolbox.filter_dict_for_none(d)
+        self.assertEquals(filtered_dict, result_dict)
+

src/generic/toolbox.py

@@ -301,3 +301,17 @@ def merge_dicts(dict1, dict2):
             if key not in copy_dict1:
                 copy_dict1[key] = value
         return copy_dict1
+
+    @staticmethod
+    def filter_dict_for_none(my_dict):
+        my_dict_copy = my_dict.copy()
+        for key, value in my_dict.iteritems():
+            if isinstance(value, dict):
+                my_dict_copy[key] = ExtensionsToolbox.filter_dict_for_none(value)
+                if my_dict_copy[key] == {}:
+                    my_dict_copy.pop(key)
+                continue
+            if value is None:
+                my_dict_copy.pop(key)
+
+        return my_dict_copy