From 5b499c156930495973b7201c04fedb21f3cf6550 Mon Sep 17 00:00:00 2001 From: gve Date: Sat, 20 Apr 2019 17:21:27 +0200 Subject: [PATCH] add blacked files --- vaultify/__init__.py | 7 +-- vaultify/base.py | 16 +++--- vaultify/config.py | 85 ++++++++++-------------------- vaultify/consumers.py | 99 ++++++++++++----------------------- vaultify/exceptions.py | 5 ++ vaultify/providers.py | 114 ++++++++++++++++++++--------------------- vaultify/util.py | 52 ++++++++----------- vaultify/vaultify.py | 51 +++++------------- 8 files changed, 168 insertions(+), 261 deletions(-) diff --git a/vaultify/__init__.py b/vaultify/__init__.py index e682044..edde7c2 100644 --- a/vaultify/__init__.py +++ b/vaultify/__init__.py @@ -4,11 +4,8 @@ CFG = configure() logging.config.dictConfig(CFG) logger = logging.getLogger(__name__) -logger.debug('vaultify module initializing...') +logger.debug("vaultify module initializing...") from .vaultify import main -__all__ = ( - "main" - "CFG" -) \ No newline at end of file +__all__ = "main" "CFG" diff --git a/vaultify/base.py b/vaultify/base.py index 6d1868e..9e5c304 100644 --- a/vaultify/base.py +++ b/vaultify/base.py @@ -13,21 +13,19 @@ class Provider: def __str__(self): - return '{}'.format(self.__class__) + return "{}".format(self.__class__) @abc.abstractmethod def get_secrets(self) -> dict: pass - class Consumer(metaclass=abc.ABCMeta): def __str__(self): - return '{}'.format(self.__class__) + return "{}".format(self.__class__) @abc.abstractmethod - def consume_secrets(self, - data: dict) -> bool: + def consume_secrets(self, data: dict) -> bool: pass @@ -36,9 +34,8 @@ class API(metaclass=abc.ABCMeta): ABC meta class for later extensibility and cheap NotImplementedErrors """ - def __init__(self, - provider: Provider, - consumer: Consumer): + + def __init__(self, provider: Provider, consumer: Consumer): self._provider = provider self._consumer = consumer @@ -52,8 +49,7 @@ def get_secrets(self) -> dict: pass @abc.abstractmethod - def consume_secrets(self, - data: dict) -> bool: + def consume_secrets(self, data: dict) -> bool: pass @abc.abstractmethod diff --git a/vaultify/config.py b/vaultify/config.py index 4176c4d..e07f3da 100644 --- a/vaultify/config.py +++ b/vaultify/config.py @@ -4,32 +4,21 @@ from pprint import pprint from .util import yaml_dict_merge, load_yaml_cfg_sources -MODULE_BASE_DIR = os.path.dirname( - os.path.realpath(__file__)) -ETC_DEFAULT_CONFIG = '/etc/default/vaultify.yml' -USER_CONFIG = '{}/.vaultify.yml'.format(os.environ.get("HOME")) -LOCAL_CONFIG = '{}/.vaultify.yml'.format(os.environ.get("PWD", '.')) +MODULE_BASE_DIR = os.path.dirname(os.path.realpath(__file__)) +ETC_DEFAULT_CONFIG = "/etc/default/vaultify.yml" +USER_CONFIG = "{}/.vaultify.yml".format(os.environ.get("HOME")) +LOCAL_CONFIG = "{}/.vaultify.yml".format(os.environ.get("PWD", ".")) -CFG_DEFAULT_FILES = [ - ETC_DEFAULT_CONFIG, - USER_CONFIG, - LOCAL_CONFIG] +CFG_DEFAULT_FILES = [ETC_DEFAULT_CONFIG, USER_CONFIG, LOCAL_CONFIG] BASE_CFG = { - "vaultify": { - }, + "vaultify": {}, "handlers": { - "console": { - "level": "WARN", - }, - "file": { - "level": "WARN", - "filename": "./vaultify.log" - } + "console": {"level": "WARN"}, + "file": {"level": "WARN", "filename": "./vaultify.log"}, }, - "loggers": { - } + "loggers": {}, } @@ -38,36 +27,31 @@ "formatters": { "simple": { "class": "logging.Formatter", - "format": '[%(levelname)-5.5s] [%(name)-20.20s] - %(message)s' + "format": "[%(levelname)-5.5s] [%(name)-20.20s] - %(message)s", }, "extended": { "class": "logging.Formatter", - "format": "[%(asctime)s] [%(name)-20.20s] [%(levelname)-5.5s] %(message)s" - } + "format": "[%(asctime)s] [%(name)-20.20s] [%(levelname)-5.5s] %(message)s", + }, }, "handlers": { "console": { "class": "logging.StreamHandler", "level": "DEBUG", - "stream": 'ext://sys.stdout', - "formatter": "simple" + "stream": "ext://sys.stdout", + "formatter": "simple", }, "file": { "class": "logging.FileHandler", "level": "DEBUG", "filename": "./debug.log", "mode": "w", - "formatter": "extended" - } + "formatter": "extended", + }, }, "loggers": { - "": { - "level": "DEBUG", - "handlers": ["console", "file"], - "propagate": True - }, - - } + "": {"level": "DEBUG", "handlers": ["console", "file"], "propagate": True} + }, } @@ -75,36 +59,25 @@ "vaultify": { "provider": { "class": os.environ.get("VAULTIFY_PROVIDER"), - "args": { - "secret": os.environ.get("VAULTIFY_SECRET") - } + "args": {"secret": os.environ.get("VAULTIFY_SECRET")}, }, "consumer": { "class": os.environ.get("VAULTIFY_CONSUMER"), - "args": { - "path": os.environ.get("VAULTIFY_DESTINATION") - } - } + "args": {"path": os.environ.get("VAULTIFY_DESTINATION")}, + }, }, "handlers": { - "console": { - "level": os.environ.get("VAULTIFY_LOG_LEVEL"), - }, + "console": {"level": os.environ.get("VAULTIFY_LOG_LEVEL")}, "file": { "level": os.environ.get("VAULTIFY_LOG_LEVEL"), - "filename": os.environ.get("VAULTIFY_LOG_FILE") - } + "filename": os.environ.get("VAULTIFY_LOG_FILE"), + }, }, - "loggers": { - "": { - "level": os.environ.get("VAULTIFY_LOG_LEVEL") - } - } + "loggers": {"": {"level": os.environ.get("VAULTIFY_LOG_LEVEL")}}, } -def configure( - yaml_files: list = CFG_DEFAULT_FILES) -> dict: +def configure(yaml_files: list = CFG_DEFAULT_FILES) -> dict: """ This populates the global config dictionary with merged values @@ -124,9 +97,7 @@ def configure( for src in cfg_sources: if src: - config_data = yaml_dict_merge( - config_data, src - ) + config_data = yaml_dict_merge(config_data, src) config_data = yaml_dict_merge(config_data, ENV_CFG) @@ -141,5 +112,5 @@ def configure( "CFG_DEFAULT_FILES", "BASE_CFG", "LOG_CFG", - "configure" + "configure", ) diff --git a/vaultify/consumers.py b/vaultify/consumers.py index bb346f1..93757af 100644 --- a/vaultify/consumers.py +++ b/vaultify/consumers.py @@ -15,11 +15,7 @@ from .base import Consumer -__all__ = ( - 'DotEnvWriter', - 'JsonWriter', - 'EnvRunner' -) +__all__ = ("DotEnvWriter", "JsonWriter", "EnvRunner") logger = logging.getLogger(__name__) @@ -31,26 +27,21 @@ class FileWriter: >>> isinstance(fw, FileWriter) True """ - def __init__(self, - path: str, - mode: oct = 0o600, - overwrite: bool = False, - *args, **kwargs): + + def __init__( + self, path: str, mode: oct = 0o600, overwrite: bool = False, *args, **kwargs + ): self.path = path self.mode = mode self.overwrite = overwrite def _write_data_to_fd(self, data: str): - with open(os.open(self.path, - os.O_CREAT | os.O_WRONLY, - 0o200), 'w') as file_out: - logger.info( - "writing to {}, mode {}".format( - self.path, oct(self.mode))) - + with open(os.open(self.path, os.O_CREAT | os.O_WRONLY, 0o200), "w") as file_out: + logger.info("writing to {}, mode {}".format(self.path, oct(self.mode))) + file_out.write(data) - file_out.write('\n') - + file_out.write("\n") + def write(self, data: str): """ >>> fw = FileWriter('tests/new.filewriter', overwrite=False) @@ -70,16 +61,12 @@ def write(self, data: str): self._write_data_to_fd(data) else: if self.overwrite: - logger.warning( - 'overwriting {}'.format(self.path)) + logger.warning("overwriting {}".format(self.path)) self._write_data_to_fd(data) else: - logger.warning( - '{} already exists: skip'.format( - self.path)) - - os.chmod( - self.path, self.mode) + logger.warning("{} already exists: skip".format(self.path)) + + os.chmod(self.path, self.mode) class DotEnvWriter(Consumer, FileWriter): @@ -91,11 +78,10 @@ class DotEnvWriter(Consumer, FileWriter): >>> open('tests/new.env').read() "export K1='V1'\\nexport K2='V2'\\n" """ + def consume_secrets(self, data: dict): - self.write( - "\n".join( - util.dict2env(data))) - + self.write("\n".join(util.dict2env(data))) + class JsonWriter(Consumer, FileWriter): """ @@ -105,13 +91,11 @@ class JsonWriter(Consumer, FileWriter): >>> open('tests/new.json').read() '{\\n "K1": "V1",\\n "K2": "V2"\\n}\\n' """ + def consume_secrets(self, data: dict): - self.write(json.dumps( - data, - sort_keys=True, - indent=2) - ) - + self.write(json.dumps(data, sort_keys=True, indent=2)) + + class YamlWriter(Consumer, FileWriter): """ This Consumer writes secrets as a YAML dictionary @@ -120,12 +104,12 @@ class YamlWriter(Consumer, FileWriter): >>> open('tests/new.yaml').read() 'K1: V1\\nK2: V2\\n\\n' """ + def consume_secrets(self, data: dict): - self.write(yaml.dump( - data, - default_flow_style=False, - allow_unicode=True, - encoding='utf-8').decode() + self.write( + yaml.dump( + data, default_flow_style=False, allow_unicode=True, encoding="utf-8" + ).decode() ) @@ -147,39 +131,24 @@ class EnvRunner(Consumer): ... FileNotFoundError: [Errno 2] No such file or directory: 'nowhere.sh': 'nowhere.sh' """ - + def __init__(self, path: str): - self.path = os.environ.get( - "VAULTIFY_TARGET", path - ).split() + self.path = os.environ.get("VAULTIFY_TARGET", path).split() def consume_secrets(self, data: dict): prepared_env = dict(os.environ) for key, value in data.items(): - prepared_env.update( - {key: value} - ) - logger.info( - '{} enriched the environment'.format(self)) + prepared_env.update({key: value}) + logger.info("{} enriched the environment".format(self)) try: # TODO Overhaul this - proc = run( - self.path, - stdout=PIPE, - stderr=PIPE, - env=prepared_env - ) - logger.info( - 'running the process "{}"'.format(self.path)) + proc = run(self.path, stdout=PIPE, stderr=PIPE, env=prepared_env) + logger.info('running the process "{}"'.format(self.path)) except FileNotFoundError as error: - logger.critical( - 'error in {} executing "{}"'.format(self, self.path) - ) + logger.critical('error in {} executing "{}"'.format(self, self.path)) raise error - print( - proc.stdout.decode() - ) + print(proc.stdout.decode()) diff --git a/vaultify/exceptions.py b/vaultify/exceptions.py index 94d6d88..a274a82 100644 --- a/vaultify/exceptions.py +++ b/vaultify/exceptions.py @@ -5,6 +5,7 @@ class ValidationError(Exception): def __str___(self): return "{} did not validate successfully".format(self.args) + """ Serves as the common base class for vaultify validation problems. """ @@ -15,6 +16,7 @@ class VaultifyEnvironmentError(ValidationError): """ Raise this when any vaultify environment variable is unset. """ + pass @@ -22,6 +24,7 @@ class AdapterValidationError(ValidationError): """ Raise this when any vaultify adapter is misconfigured """ + pass @@ -29,6 +32,7 @@ class ProviderError(AdapterValidationError): """ Raise this when a Provider is misconfigured """ + pass @@ -36,4 +40,5 @@ class ConsumerError(AdapterValidationError): """ Raise this when a Consumer is misconfigured """ + pass diff --git a/vaultify/providers.py b/vaultify/providers.py index 0bf2690..410ab8e 100644 --- a/vaultify/providers.py +++ b/vaultify/providers.py @@ -15,34 +15,27 @@ logger = logging.getLogger(__name__) -__all__ = ( - 'VaultProvider', - 'GPGProvider', - 'OpenSSLProvider', - 'PlainTextProvider' -) +__all__ = ("VaultProvider", "GPGProvider", "OpenSSLProvider", "PlainTextProvider") class VaultProvider(Provider): """ This is the original Provider which uses HashiCorp Vault to fetch secrets. """ + def __init__( - self, - paths: str = None, - token: str = os.environ.get("VAULTIFY_SECRET"), - addr: str = None + self, + paths: str = None, + token: str = os.environ.get("VAULTIFY_SECRET"), + addr: str = None, ): self.token = os.environ.get("VAULT_TOKEN", token) self.addr = os.environ.get("VAULT_ADDR", addr) - self.paths = os.environ.get("VAULT_PATHS", paths).split(',') + self.paths = os.environ.get("VAULT_PATHS", paths).split(",") - self.client = hvac.Client( - url=self.addr, - token=self.token - ) - logger.debug('VaultProvider initialized') + self.client = hvac.Client(url=self.addr, token=self.token) + logger.debug("VaultProvider initialized") def get_secrets(self): """ @@ -52,8 +45,7 @@ def get_secrets(self): secrets = {} for path in self.paths: secrets[path] = self.client.read(path)["data"] - logger.info( - 'provided secrets from {}'.format(path)) + logger.info("provided secrets from {}".format(path)) return secrets @@ -65,22 +57,20 @@ class OpenSSLProvider(Provider): >>> OpenSSLProvider(secret='abc').get_secrets() {'./assets/test.enc': {'K1': 'V1', 'K2': 'V2'}} """ - def __init__(self, - secret: str, - cipher: str='aes-256-cbc', - md: str='sha256'): + + def __init__(self, secret: str, cipher: str = "aes-256-cbc", md: str = "sha256"): self.secret = secret self.cipher = cipher self.md = md self.popen_kwargs = dict( bufsize=-1, - executable='/usr/bin/openssl', + executable="/usr/bin/openssl", universal_newlines=True, - encoding='utf-8', + encoding="utf-8", stderr=PIPE, - stdout=PIPE - ) - logger.debug('OpenSSLProvider initialized') + stdout=PIPE, + ) + logger.debug("OpenSSLProvider initialized") def get_secrets(self): """ @@ -90,19 +80,25 @@ def get_secrets(self): """ secrets = {} - for filename in glob.glob('./assets/*.enc'): + for filename in glob.glob("./assets/*.enc"): out = run_process( - ['openssl', self.cipher, - '-d', '-a', - '-md', self.md, - '-in', filename, - '-k', self.secret], - self.popen_kwargs + [ + "openssl", + self.cipher, + "-d", + "-a", + "-md", + self.md, + "-in", + filename, + "-k", + self.secret, + ], + self.popen_kwargs, ) secrets[filename] = env2dict(out) - logger.info( - 'provided secrets from {}'.format(filename)) + logger.info("provided secrets from {}".format(filename)) return secrets @@ -112,17 +108,18 @@ class GPGProvider(Provider): >>> GPGProvider(secret='abc').get_secrets() {'./assets/test.gpg': {'K1': 'V1', 'K2': 'V2'}} """ + def __init__(self, secret: str): # nosec self.secret = secret self.popen_kwargs = dict( bufsize=-1, - executable='/usr/bin/gpg', + executable="/usr/bin/gpg", universal_newlines=True, - encoding='utf-8', + encoding="utf-8", stderr=PIPE, - stdout=PIPE - ) - logger.debug('GPGProvider initialised') + stdout=PIPE, + ) + logger.debug("GPGProvider initialised") def get_secrets(self): """ @@ -130,36 +127,39 @@ def get_secrets(self): to run a command equivalent to `gpg -qd ` """ secrets = {} - for filename in glob.glob('./assets/*.gpg'): + for filename in glob.glob("./assets/*.gpg"): out = run_process( - ['gpg', '-qd', - '--yes', '--batch', - '--passphrase={}'.format(self.secret), - filename], - self.popen_kwargs - ) + [ + "gpg", + "-qd", + "--yes", + "--batch", + "--passphrase={}".format(self.secret), + filename, + ], + self.popen_kwargs, + ) secrets[filename] = env2dict(out) - logger.info( - 'provided secrets from {}'.format(filename)) + logger.info("provided secrets from {}".format(filename)) return secrets - + class PlainTextProvider(Provider): """ >>> PlainTextProvider().get_secrets() {'./assets/secrets.plain': {'K1': 'V1', 'K2': 'V2'}} """ + def __init__(self): - logger.debug('GPGProvider initialised') - + logger.debug("GPGProvider initialised") + def get_secrets(self): secrets = {} - for filename in glob.glob('./assets/*.plain'): - with open(filename, 'r') as infile: + for filename in glob.glob("./assets/*.plain"): + with open(filename, "r") as infile: out = infile.read() secrets[filename] = env2dict(out) - logger.info( - 'provided secrets from {}'.format(filename)) + logger.info("provided secrets from {}".format(filename)) return secrets diff --git a/vaultify/util.py b/vaultify/util.py index 9131a7e..e48f69e 100644 --- a/vaultify/util.py +++ b/vaultify/util.py @@ -24,12 +24,8 @@ def dict2env(secret_data: dict) -> t.Iterable: ["export KEY1='VAL1'", "export KEY2='VAL2'"] """ - logger.debug( - "transforming this dict to newline separated K=V pairs") - return [ - "export {}='{}'".format(key, value) - for key, value in secret_data.items() - ] + logger.debug("transforming this dict to newline separated K=V pairs") + return ["export {}='{}'".format(key, value) for key, value in secret_data.items()] def env2dict(env_data: t.AnyStr) -> dict: @@ -48,15 +44,14 @@ def env2dict(env_data: t.AnyStr) -> dict: >>> env2dict('KEY1= #VAL1\\n#KEY2=VAL2') {'KEY1': ''} """ - logger.debug( - "transforming the env to dict-class") + logger.debug("transforming the env to dict-class") dict_data = {} - line_data = env_data.split('\n') + line_data = env_data.split("\n") for line in line_data: line = re.sub("\s*#.*", "", line) if line: - key, value = line.split('=') + key, value = line.split("=") dict_data[key] = value return dict_data @@ -75,23 +70,21 @@ def mask_secrets(secrets: dict) -> dict: ... 4: "abc"}}}) {'path': {'a': '***', 'b': '***', 'c': {3: '***', 4: '***'}}} """ - logger.debug( - "hiding secrets for logs") + logger.debug("hiding secrets for logs") masked = {} for key, value in secrets.items(): if isinstance(value, (str, int)): # being extra destructive here, since we do # never want secrets leaked into logs - value = '***' + value = "***" elif isinstance(value, dict): value = mask_secrets(value) masked[key] = value return masked -def run_process(cmd: t.Union[list, tuple], - kwargs: dict) -> t.AnyStr: +def run_process(cmd: t.Union[list, tuple], kwargs: dict) -> t.AnyStr: """ Run a target process with Popen and kwargs @@ -107,7 +100,8 @@ def run_process(cmd: t.Union[list, tuple], if proc.returncode: # if there is non zero rc, please die raise ChildProcessError( - 'terminated with an non-zero value: {}'.format(proc.stderr.read())) + "terminated with an non-zero value: {}".format(proc.stderr.read()) + ) except OSError as error: # this case should handle a missing/non-executable binary @@ -149,12 +143,14 @@ def yaml_dict_merge(a: dict, b: dict) -> dict: """ key = None try: - if (a is None - or isinstance(a, str) - or isinstance(a, int) - or isinstance(a, float) - or isinstance(a, bool)): - #^ border case for first run or if a is a primitive + if ( + a is None + or isinstance(a, str) + or isinstance(a, int) + or isinstance(a, float) + or isinstance(a, bool) + ): + # ^ border case for first run or if a is a primitive if b: # override a only when b has value != None a = b @@ -181,7 +177,8 @@ def yaml_dict_merge(a: dict, b: dict) -> dict: a[key] = b[key] else: raise ValueError( - 'Cannot merge non-dict "{}" with dict "{}"'.format(b, a)) + 'Cannot merge non-dict "{}" with dict "{}"'.format(b, a) + ) else: raise NotImplementedError( 'Merging "{}" with "{}" is not implemented.'.format(type(a), type(b)) @@ -202,15 +199,10 @@ def load_yaml_cfg_sources(yaml_files: t.Iterable) -> list: if os.path.isfile(config_file): with open(config_file) as yaml_conf: # linter.run(LINT_CONF, yaml_conf) - logger.debug('reading %s', config_file) - cfg_sources.append( - yaml.safe_load(yaml_conf) - ) + logger.debug("reading %s", config_file) + cfg_sources.append(yaml.safe_load(yaml_conf)) return cfg_sources def prefer_env_if_not_none(key: str) -> t.Union[str, None]: return os.environ.get(key, None) - - - diff --git a/vaultify/vaultify.py b/vaultify/vaultify.py index 490ea61..2d38792 100644 --- a/vaultify/vaultify.py +++ b/vaultify/vaultify.py @@ -38,18 +38,11 @@ class Vaultify(API): """ def get_secrets(self) -> dict: - logger.info( - 'providing secrets from {}'.format( - self._provider) - ) + logger.info("providing secrets from {}".format(self._provider)) return self._provider.get_secrets() - def consume_secrets(self, - data: dict) -> bool: - logger.info( - 'consuming secrets with {}'.format( - self._consumer) - ) + def consume_secrets(self, data: dict) -> bool: + logger.info("consuming secrets with {}".format(self._consumer)) return self._consumer.consume_secrets(data) def validate(self) -> t.Iterable: @@ -64,20 +57,16 @@ def validate(self) -> t.Iterable: """ results = [] - if not isinstance(self._provider, - Provider): + if not isinstance(self._provider, Provider): results.append( ProviderError( - "The Provider {} is not a Provider".format( - self._provider) + "The Provider {} is not a Provider".format(self._provider) ) ) - if not isinstance(self._consumer, - Consumer): + if not isinstance(self._consumer, Consumer): results.append( ConsumerError( - "The Consumer {} is not a Consumer".format( - self._consumer) + "The Consumer {} is not a Consumer".format(self._consumer) ) ) @@ -87,16 +76,12 @@ def run(self) -> bool: secrets = self.get_secrets() if not secrets: raise ValueError( - 'The provider did not yield anything: {}'.format( - self._provider) + "The provider did not yield anything: {}".format(self._provider) ) to_consumer = {} for data in secrets.values(): - logger.info( - 'consuming secret: %s', - mask_secrets(secrets) - ) + logger.info("consuming secret: %s", mask_secrets(secrets)) to_consumer.update(data) return self.consume_secrets(to_consumer) @@ -119,24 +104,18 @@ def factory(config_dict: dict, **kwargs) -> Vaultify: """ logger.debug("factory starting..") - vfy = config_dict['vaultify'] + vfy = config_dict["vaultify"] from . import providers from . import consumers - provider_class = getattr( - providers, - vfy["provider"]["class"] - ) - consumer_class = getattr( - consumers, - vfy["consumer"]["class"] - ) + provider_class = getattr(providers, vfy["provider"]["class"]) + consumer_class = getattr(consumers, vfy["consumer"]["class"]) return Vaultify( provider=provider_class(**vfy["provider"]["args"]), - consumer=consumer_class(**vfy["consumer"]["args"]) - ) + consumer=consumer_class(**vfy["consumer"]["args"]), + ) def main() -> None: @@ -148,5 +127,3 @@ def main() -> None: vaultify = factory(CFG) vaultify.validate() vaultify.run() - -