From bcb795a663e1cb141aeae47c13314be70b8fac5c Mon Sep 17 00:00:00 2001 From: Rapejim Date: Sun, 11 Oct 2020 11:15:07 +0200 Subject: [PATCH 01/11] feature: Added namespace to environment variables and legacy versions --- pyms/constants.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyms/constants.py b/pyms/constants.py index 5a70a6b..79dc147 100644 --- a/pyms/constants.py +++ b/pyms/constants.py @@ -1,8 +1,10 @@ -CONFIGMAP_FILE_ENVIRONMENT = "CONFIGMAP_FILE" +CONFIGMAP_FILE_ENVIRONMENT = "PYMS_CONFIGMAP_FILE" +CONFIGMAP_FILE_ENVIRONMENT_LEGACY = "CONFIGMAP_FILE" DEFAULT_CONFIGMAP_FILENAME = "config.yml" -CRYPT_FILE_KEY_ENVIRONMENT = "KEY_FILE" +CRYPT_FILE_KEY_ENVIRONMENT = "PYMS_KEY_FILE" +CRYPT_FILE_KEY_ENVIRONMENT_LEGACY = "KEY_FILE" DEFAULT_KEY_FILENAME = "key.key" From 1a84f2ed371c469a2f2fc63c619b974da874f022 Mon Sep 17 00:00:00 2001 From: Rapejim Date: Sun, 11 Oct 2020 11:16:27 +0200 Subject: [PATCH 02/11] feature: Added warnings at use case of old environment variables names --- pyms/config/conf.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/pyms/config/conf.py b/pyms/config/conf.py index 737c37d..fcfe6ba 100644 --- a/pyms/config/conf.py +++ b/pyms/config/conf.py @@ -1,7 +1,13 @@ +import logging +import os + from pyms.config.confile import ConfFile -from pyms.constants import PYMS_CONFIG_WHITELIST_KEYWORDS +from pyms.constants import PYMS_CONFIG_WHITELIST_KEYWORDS, CONFIGMAP_FILE_ENVIRONMENT_LEGACY, \ + CONFIGMAP_FILE_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY, LOGGER_NAME from pyms.exceptions import ServiceDoesNotExistException, ConfigErrorException, AttrDoesNotExistException +logger = logging.getLogger(LOGGER_NAME) + def get_conf(*args, **kwargs): """ @@ -42,6 +48,9 @@ def get_conf(*args, **kwargs): def validate_conf(*args, **kwargs): + # TODO Remove temporally deprecated warnings on future versions + __verify_deprecated_env_variables() + config = ConfFile(*args, **kwargs) is_config_ok = True try: @@ -104,3 +113,21 @@ def validate_conf(*args, **kwargs): config: DEBUG: true TESTING: true""".format(wrong_keywords)) + + +def __verify_deprecated_env_variables(): + if os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is not None: + if os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is not None: + logger.warning("If you are using {} environment variable, {} value will be ignored" + .format(CONFIGMAP_FILE_ENVIRONMENT, CONFIGMAP_FILE_ENVIRONMENT_LEGACY)) + else: + logger.warning("{} environment variable is deprecated on this version, you must use {} instead" + .format(CONFIGMAP_FILE_ENVIRONMENT, CONFIGMAP_FILE_ENVIRONMENT_LEGACY)) + + if os.getenv(CRYPT_FILE_KEY_ENVIRONMENT_LEGACY) is not None: + if os.getenv(CRYPT_FILE_KEY_ENVIRONMENT) is not None: + logger.warning("If you are using {} environment variable, {} value will be ignored" + .format(CRYPT_FILE_KEY_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY)) + else: + logger.warning("{} environment variable is deprecated on this version, you must use {} instead" + .format(CONFIGMAP_FILE_ENVIRONMENT, CONFIGMAP_FILE_ENVIRONMENT_LEGACY)) From b13fd0d89fe4a514b210beff91b9e999e5824fcc Mon Sep 17 00:00:00 2001 From: Rapejim Date: Sun, 11 Oct 2020 11:21:03 +0200 Subject: [PATCH 03/11] feature: Added backward compatibility at use case of old environment variables names --- pyms/config/confile.py | 16 ++++++++++++++-- pyms/crypt/fernet.py | 17 ++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/pyms/config/confile.py b/pyms/config/confile.py index 045cd03..ac63d01 100644 --- a/pyms/config/confile.py +++ b/pyms/config/confile.py @@ -1,11 +1,13 @@ """Module to read yaml or json conf""" import logging +import os import re from typing import Dict, Union, Text, Tuple, Iterable import anyconfig -from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT, LOGGER_NAME, DEFAULT_CONFIGMAP_FILENAME +from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT, LOGGER_NAME, DEFAULT_CONFIGMAP_FILENAME, \ + CONFIGMAP_FILE_ENVIRONMENT_LEGACY from pyms.exceptions import AttrDoesNotExistException, ConfigDoesNotFoundException from pyms.utils.files import LoadFile @@ -34,7 +36,10 @@ def __init__(self, *args, **kwargs): self[key] = getattr(obj, key) ``` """ - self._loader = LoadFile(kwargs.get("path"), CONFIGMAP_FILE_ENVIRONMENT, DEFAULT_CONFIGMAP_FILENAME) + # TODO Remove temporally backward compatibility on future versions + configmap_file_env = self.__get_updated_configmap_file_env() # Temporally backward compatibility + + self._loader = LoadFile(kwargs.get("path"), configmap_file_env, DEFAULT_CONFIGMAP_FILENAME) self._crypt_cls = kwargs.get("crypt") if self._crypt_cls: self._crypt = self._crypt_cls(path=kwargs.get("path")) @@ -125,3 +130,10 @@ def reload(self): def __setattr__(self, name, value, *args, **kwargs): super().__setattr__(name, value) + + @staticmethod + def __get_updated_configmap_file_env() -> str: + if (os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is None) & (os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is not None): + return CONFIGMAP_FILE_ENVIRONMENT_LEGACY + else: + return CONFIGMAP_FILE_ENVIRONMENT diff --git a/pyms/crypt/fernet.py b/pyms/crypt/fernet.py index 0d63ab4..79acc56 100644 --- a/pyms/crypt/fernet.py +++ b/pyms/crypt/fernet.py @@ -7,7 +7,7 @@ from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC -from pyms.constants import CRYPT_FILE_KEY_ENVIRONMENT, DEFAULT_KEY_FILENAME +from pyms.constants import CRYPT_FILE_KEY_ENVIRONMENT, DEFAULT_KEY_FILENAME, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY from pyms.crypt.driver import CryptAbstract from pyms.exceptions import FileDoesNotExistException from pyms.utils.files import LoadFile @@ -15,7 +15,9 @@ class Crypt(CryptAbstract): def __init__(self, *args, **kwargs): - self._loader = LoadFile(kwargs.get("path"), CRYPT_FILE_KEY_ENVIRONMENT, DEFAULT_KEY_FILENAME) + # TODO Remove temporally backward compatibility on future versions + crypt_file_key_env = self.__get_updated_crypt_file_key_env() # Temporally backward compatibility + self._loader = LoadFile(kwargs.get("path"), crypt_file_key_env, DEFAULT_KEY_FILENAME) super().__init__(*args, **kwargs) def generate_key(self, password: Text, write_to_file: bool = False) -> bytes: @@ -36,9 +38,11 @@ def generate_key(self, password: Text, write_to_file: bool = False) -> bytes: def read_key(self): key = self._loader.get_file() if not key: + # TODO Remove temporally backward compatibility on future versions + crypt_file_key_env = self.__get_updated_crypt_file_key_env() # Temporally backward compatibility raise FileDoesNotExistException( "Decrypt key {} not exists. You must set a correct env var {} " - "or run `pyms crypt create-key` command".format(self._loader.path, CRYPT_FILE_KEY_ENVIRONMENT)) + "or run `pyms crypt create-key` command".format(self._loader.path, crypt_file_key_env)) return key def encrypt(self, message): @@ -57,3 +61,10 @@ def decrypt(self, encrypted): def delete_key(self): os.remove(self._loader.get_path_from_env()) + + @staticmethod + def __get_updated_crypt_file_key_env() -> str: + if (os.getenv(CRYPT_FILE_KEY_ENVIRONMENT_LEGACY) is None) & (os.getenv(CRYPT_FILE_KEY_ENVIRONMENT) is not None): + return CRYPT_FILE_KEY_ENVIRONMENT_LEGACY + else: + return CRYPT_FILE_KEY_ENVIRONMENT From 1bac2367d5c563baa494d64122555c94e5583b35 Mon Sep 17 00:00:00 2001 From: Rapejim Date: Sun, 11 Oct 2020 11:23:06 +0200 Subject: [PATCH 04/11] docs: Added namespace to environment variables on comments and docs --- docs/configuration.md | 8 ++++---- docs/encrypt_decryt_configuration.md | 4 ++-- docs/ms_class.md | 2 +- examples/mininum_microservice_docker/Dockerfile | 2 +- pyms/config/confile.py | 4 ++-- pyms/flask/app/create_app.py | 2 +- tests/test_cmd.py | 2 +- tests/test_crypt.py | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 016f596..c259e97 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,9 +2,9 @@ ## Environments variables of PyMS: -**CONFIGMAP_FILE**: The path to the configuration file. By default, PyMS searches for the configuration file in your +**PYMS_CONFIGMAP_FILE**: The path to the configuration file. By default, PyMS searches for the configuration file in your current folder with the name "config.yml" -**KEY_FILE**: The path to the key file to decrypt your configuration. By default, PyMS searches for the configuration file in your +**PYMS_KEY_FILE**: The path to the key file to decrypt your configuration. By default, PyMS searches for the configuration file in your current folder with the name "key.key" ## Create configuration @@ -136,7 +136,7 @@ API = Api( ## Looking for Configuration file with Kubernetes Configmaps By default, the Microservice class searches for a config.yml in the same path. You can set a different route or set a json file. -To change this path, you must define an environment variable called `CONFIGMAP_FILE`. +To change this path, you must define an environment variable called `PYMS_CONFIGMAP_FILE`. This way of looking for the configuration is useful when you work with Docker and Kubernetes. For example, you could integrate a configmap of Kubernetes, with this microservice and a deployment with: @@ -154,7 +154,7 @@ spec: - name: my-microservice image: ... env: - - name: CONFIGMAP_FILE + - name: PYMS_CONFIGMAP_FILE value: "/usr/share/microservice/config.yaml" volumeMounts: diff --git a/docs/encrypt_decryt_configuration.md b/docs/encrypt_decryt_configuration.md index b608c3d..ebdb827 100644 --- a/docs/encrypt_decryt_configuration.md +++ b/docs/encrypt_decryt_configuration.md @@ -57,7 +57,7 @@ Move your key, for example, to `mv key.key /home/my_user/keys/myproject.key` then, store the key in a environment variable with: ```bash -export KEY_FILE=/home/my_user/keys/myproject.key +export PYMS_KEY_FILE=/home/my_user/keys/myproject.key ``` ## 3. Encrypt your information and store it in config @@ -89,7 +89,7 @@ can find the answer ## 4. Decrypt from your config file Pyms knows if a variable is encrypted if this var start with the prefix `enc_` or `ENC_`. PyMS searches for your key file -in the `KEY_FILE` env variable and decrypts this value to store it in the same variable without the `enc_` prefix, +in the `PYMS_KEY_FILE` env variable and decrypts this value to store it in the same variable without the `enc_` prefix, for example, ```yaml diff --git a/docs/ms_class.md b/docs/ms_class.md index 795d2f3..e541537 100644 --- a/docs/ms_class.md +++ b/docs/ms_class.md @@ -33,7 +33,7 @@ pyms: Check the [Configuration](configuration.md) section to learn how to create a configuration file. The `Microservice` class searches for a `config.yml` in the directory you pass in `path` parameter or looks for the file in -`CONFIGMAP_FILE` env var. +`PYMS_CONFIGMAP_FILE` env var. Each keyword in our configuration block, can be accessed in our Microservice object through the attribute `config`. diff --git a/examples/mininum_microservice_docker/Dockerfile b/examples/mininum_microservice_docker/Dockerfile index 83fa09c..2f89258 100644 --- a/examples/mininum_microservice_docker/Dockerfile +++ b/examples/mininum_microservice_docker/Dockerfile @@ -4,7 +4,7 @@ RUN apk add --update curl gcc g++ git libffi-dev openssl-dev python3-dev build-b && rm -rf /var/cache/apk/* ENV PYTHONUNBUFFERED=1 APP_HOME=/microservice/ -ENV CONFIGMAP_FILE="$APP_HOME"config-docker.yml +ENV PYMS_CONFIGMAP_FILE="$APP_HOME"config-docker.yml RUN mkdir $APP_HOME && adduser -S -D -H python RUN chown -R python $APP_HOME diff --git a/pyms/config/confile.py b/pyms/config/confile.py index ac63d01..1d9c605 100644 --- a/pyms/config/confile.py +++ b/pyms/config/confile.py @@ -16,7 +16,7 @@ class ConfFile(dict): """Recursive get configuration from dictionary, a config file in JSON or YAML format from a path or - `CONFIGMAP_FILE` environment variable. + `PYMS_CONFIGMAP_FILE` environment variable. **Atributes:** * path: Path to find the `DEFAULT_CONFIGMAP_FILENAME` and `DEFAULT_KEY_FILENAME` if use encrypted vars * empty_init: Allow blank variables @@ -28,7 +28,7 @@ class ConfFile(dict): def __init__(self, *args, **kwargs): """ Get configuration from a dictionary(variable `config`), from path (variable `path`) or from - environment with the constant `CONFIGMAP_FILE` + environment with the constant `PYMS_CONFIGMAP_FILE` Set the configuration as upper case to inject the keys in flask config. Flask search for uppercase keys in `app.config.from_object` ```python diff --git a/pyms/flask/app/create_app.py b/pyms/flask/app/create_app.py index 74df253..14bfdae 100644 --- a/pyms/flask/app/create_app.py +++ b/pyms/flask/app/create_app.py @@ -39,7 +39,7 @@ def example(): app.run() ``` Environments variables of PyMS: - **CONFIGMAP_FILE**: The path to the configuration file. By default, PyMS search the configuration file in your + **PYMS_CONFIGMAP_FILE**: The path to the configuration file. By default, PyMS search the configuration file in your actual folder with the name "config.yml" ## Create configuration diff --git a/tests/test_cmd.py b/tests/test_cmd.py index b43908d..049630c 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -21,7 +21,7 @@ def test_crypt_file_error(self): cmd = Command(arguments=arguments, autorun=False) with pytest.raises(FileDoesNotExistException) as excinfo: cmd.run() - assert ("Decrypt key None not exists. You must set a correct env var KEY_FILE or run " + assert ("Decrypt key None not exists. You must set a correct env var PYMS_KEY_FILE or run " "`pyms crypt create-key` command") \ in str(excinfo.value) diff --git a/tests/test_crypt.py b/tests/test_crypt.py index e3ccbbf..d4dcacf 100644 --- a/tests/test_crypt.py +++ b/tests/test_crypt.py @@ -56,7 +56,7 @@ def test_crypt_file_error(self): crypt = CryptFernet() with pytest.raises(FileDoesNotExistException) as excinfo: crypt.read_key() - assert ("Decrypt key None not exists. You must set a correct env var KEY_FILE or run " + assert ("Decrypt key None not exists. You must set a correct env var PYMS_KEY_FILE or run " "`pyms crypt create-key` command") \ in str(excinfo.value) From a1f18cd05bdb8e0e64abd93937a079db3ad4f57f Mon Sep 17 00:00:00 2001 From: Rapejim Date: Sun, 11 Oct 2020 14:42:28 +0200 Subject: [PATCH 05/11] style: changed warning messages for env. vars. deprecated --- pyms/config/conf.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyms/config/conf.py b/pyms/config/conf.py index fcfe6ba..0853e29 100644 --- a/pyms/config/conf.py +++ b/pyms/config/conf.py @@ -118,16 +118,16 @@ def validate_conf(*args, **kwargs): def __verify_deprecated_env_variables(): if os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is not None: if os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is not None: - logger.warning("If you are using {} environment variable, {} value will be ignored" + logger.warning("IMPORTANT: If you are using \"{}\" environment variable, \"{}\" value will be ignored." .format(CONFIGMAP_FILE_ENVIRONMENT, CONFIGMAP_FILE_ENVIRONMENT_LEGACY)) else: - logger.warning("{} environment variable is deprecated on this version, you must use {} instead" - .format(CONFIGMAP_FILE_ENVIRONMENT, CONFIGMAP_FILE_ENVIRONMENT_LEGACY)) + logger.warning("IMPORTANT: \"{}\" environment variable is deprecated on this version, use \"{}\" instead." + .format(CONFIGMAP_FILE_ENVIRONMENT_LEGACY, CONFIGMAP_FILE_ENVIRONMENT)) if os.getenv(CRYPT_FILE_KEY_ENVIRONMENT_LEGACY) is not None: if os.getenv(CRYPT_FILE_KEY_ENVIRONMENT) is not None: - logger.warning("If you are using {} environment variable, {} value will be ignored" + logger.warning("IMPORTANT: If you are using \"{}\" environment variable, \"{}\" value will be ignored." .format(CRYPT_FILE_KEY_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY)) else: - logger.warning("{} environment variable is deprecated on this version, you must use {} instead" - .format(CONFIGMAP_FILE_ENVIRONMENT, CONFIGMAP_FILE_ENVIRONMENT_LEGACY)) + logger.warning("IMPORTANT: \"{}\" environment variable is deprecated on this version, use \"{}\" instead." + .format(CONFIGMAP_FILE_ENVIRONMENT_LEGACY, CONFIGMAP_FILE_ENVIRONMENT)) From 1cf061e9346a98f8d053d6f6427746ff8cd3453a Mon Sep 17 00:00:00 2001 From: Rapejim Date: Mon, 12 Oct 2020 15:10:29 +0200 Subject: [PATCH 06/11] style: Added new util for apply color or style to text --- pyms/cmd/main.py | 6 +++--- pyms/config/conf.py | 35 ++++++++++++++++++++++++----------- pyms/utils/utils.py | 22 ++++++++++++++++++++++ 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/pyms/cmd/main.py b/pyms/cmd/main.py index 841c60b..eea8f0e 100755 --- a/pyms/cmd/main.py +++ b/pyms/cmd/main.py @@ -8,7 +8,7 @@ from pyms.crypt.fernet import Crypt from pyms.flask.services.swagger import merge_swagger_file -from pyms.utils import check_package_exists, import_from +from pyms.utils import check_package_exists, import_from, utils class Command: @@ -117,7 +117,7 @@ def run(self): @staticmethod def print_ok(msg=""): - print('\033[92m\033[1m ' + msg + ' \033[0m\033[0m') + print(utils.colored_text(msg, utils.Colors.BRIGHT_GREEN, True)) def print_verbose(self, msg=""): # pragma: no cover if self.verbose: @@ -125,7 +125,7 @@ def print_verbose(self, msg=""): # pragma: no cover @staticmethod def print_error(msg=""): # pragma: no cover - print('\033[91m\033[1m ' + msg + ' \033[0m\033[0m') + print(utils.colored_text(msg, utils.Colors.BRIGHT_RED, True)) def exit_with_error(self, msg=""): # pragma: no cover self.print_error(msg) diff --git a/pyms/config/conf.py b/pyms/config/conf.py index 0853e29..a15adf2 100644 --- a/pyms/config/conf.py +++ b/pyms/config/conf.py @@ -5,6 +5,7 @@ from pyms.constants import PYMS_CONFIG_WHITELIST_KEYWORDS, CONFIGMAP_FILE_ENVIRONMENT_LEGACY, \ CONFIGMAP_FILE_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY, LOGGER_NAME from pyms.exceptions import ServiceDoesNotExistException, ConfigErrorException, AttrDoesNotExistException +from pyms.utils import utils logger = logging.getLogger(LOGGER_NAME) @@ -48,8 +49,6 @@ def get_conf(*args, **kwargs): def validate_conf(*args, **kwargs): - # TODO Remove temporally deprecated warnings on future versions - __verify_deprecated_env_variables() config = ConfFile(*args, **kwargs) is_config_ok = True @@ -114,20 +113,34 @@ def validate_conf(*args, **kwargs): DEBUG: true TESTING: true""".format(wrong_keywords)) + # TODO Remove temporally deprecated warnings on future versions + __verify_deprecated_env_variables(config) + + +def __verify_deprecated_env_variables(config): + env_var_duplicated = "IMPORTANT: If you are using \"{}\" environment variable, \"{}\" value will be ignored." + env_var_deprecated = "IMPORTANT: \"{}\" environment variable is deprecated on this version, use \"{}\" instead." -def __verify_deprecated_env_variables(): if os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is not None: if os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is not None: - logger.warning("IMPORTANT: If you are using \"{}\" environment variable, \"{}\" value will be ignored." - .format(CONFIGMAP_FILE_ENVIRONMENT, CONFIGMAP_FILE_ENVIRONMENT_LEGACY)) + msg = env_var_duplicated.format(CONFIGMAP_FILE_ENVIRONMENT, CONFIGMAP_FILE_ENVIRONMENT_LEGACY) else: - logger.warning("IMPORTANT: \"{}\" environment variable is deprecated on this version, use \"{}\" instead." - .format(CONFIGMAP_FILE_ENVIRONMENT_LEGACY, CONFIGMAP_FILE_ENVIRONMENT)) + msg = env_var_deprecated.format(CONFIGMAP_FILE_ENVIRONMENT_LEGACY, CONFIGMAP_FILE_ENVIRONMENT) + try: + if config.pyms.config.DEBUG: + msg = utils.colored_text(msg, utils.Colors.BRIGHT_YELLOW, True) + except AttrDoesNotExistException: + pass + logger.warning(msg) if os.getenv(CRYPT_FILE_KEY_ENVIRONMENT_LEGACY) is not None: if os.getenv(CRYPT_FILE_KEY_ENVIRONMENT) is not None: - logger.warning("IMPORTANT: If you are using \"{}\" environment variable, \"{}\" value will be ignored." - .format(CRYPT_FILE_KEY_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY)) + msg = env_var_duplicated.format(CRYPT_FILE_KEY_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY) else: - logger.warning("IMPORTANT: \"{}\" environment variable is deprecated on this version, use \"{}\" instead." - .format(CONFIGMAP_FILE_ENVIRONMENT_LEGACY, CONFIGMAP_FILE_ENVIRONMENT)) + msg = env_var_deprecated.format(CRYPT_FILE_KEY_ENVIRONMENT_LEGACY, CRYPT_FILE_KEY_ENVIRONMENT) + try: + if config.pyms.config.DEBUG: + msg = utils.colored_text(msg, utils.Colors.BRIGHT_YELLOW, True) + except AttrDoesNotExistException: + pass + logger.warning(msg) diff --git a/pyms/utils/utils.py b/pyms/utils/utils.py index 66c17ec..af98ae4 100644 --- a/pyms/utils/utils.py +++ b/pyms/utils/utils.py @@ -5,6 +5,19 @@ from pyms.exceptions import PackageNotExists +class Colors: + BLUE = "\033[34m" + GREEN = "\033[32m" + MAGENTA = "\033[35m" + RED = "\033[31m" + YELLOW = "\033[33m" + BRIGHT_BLUE = "\033[94m" + BRIGHT_GREEN = "\033[92m" + BRIGHT_MAGENTA = "\033[95m" + BRIGHT_RED = "\033[91m" + BRIGHT_YELLOW = "\033[93m" + + def import_from(module: Text, name: Text): module = __import__(module, fromlist=[name]) return getattr(module, name) @@ -20,3 +33,12 @@ def check_package_exists(package_name: Text) -> Union[Exception, bool]: raise PackageNotExists( "{package} is not installed. try with pip install -U {package}".format(package=package_name)) return True + + +def colored_text(msg, color: Colors, bold=False): + BOLD = "\033[1m" + RESET_ALL = "\033[0m" + if bold: + return "{}{}{}{}".format(BOLD, color, msg, RESET_ALL) + else: + return "{}{}{}".format(color, msg, RESET_ALL) From 43b3d5a40de126a0be78da036302655f3493c2ba Mon Sep 17 00:00:00 2001 From: Rapejim Date: Mon, 12 Oct 2020 16:36:18 +0200 Subject: [PATCH 07/11] fix: logic for backward compatibility --- pyms/config/confile.py | 2 +- pyms/crypt/fernet.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyms/config/confile.py b/pyms/config/confile.py index 1d9c605..fb71d97 100644 --- a/pyms/config/confile.py +++ b/pyms/config/confile.py @@ -133,7 +133,7 @@ def __setattr__(self, name, value, *args, **kwargs): @staticmethod def __get_updated_configmap_file_env() -> str: - if (os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is None) & (os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is not None): + if (os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is not None) & (os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is None): return CONFIGMAP_FILE_ENVIRONMENT_LEGACY else: return CONFIGMAP_FILE_ENVIRONMENT diff --git a/pyms/crypt/fernet.py b/pyms/crypt/fernet.py index 79acc56..6a466f5 100644 --- a/pyms/crypt/fernet.py +++ b/pyms/crypt/fernet.py @@ -64,7 +64,7 @@ def delete_key(self): @staticmethod def __get_updated_crypt_file_key_env() -> str: - if (os.getenv(CRYPT_FILE_KEY_ENVIRONMENT_LEGACY) is None) & (os.getenv(CRYPT_FILE_KEY_ENVIRONMENT) is not None): + if (os.getenv(CRYPT_FILE_KEY_ENVIRONMENT_LEGACY) is not None) & (os.getenv(CRYPT_FILE_KEY_ENVIRONMENT) is None): return CRYPT_FILE_KEY_ENVIRONMENT_LEGACY else: return CRYPT_FILE_KEY_ENVIRONMENT From 44744939f02c3c3981198be2484d42e19db0f9ef Mon Sep 17 00:00:00 2001 From: Rapejim Date: Mon, 12 Oct 2020 16:58:35 +0200 Subject: [PATCH 08/11] refactor: changes for pass linter checks --- pylintrc | 3 +-- pyms/config/confile.py | 4 ++-- pyms/crypt/fernet.py | 8 ++++---- pyms/utils/utils.py | 8 +++----- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/pylintrc b/pylintrc index 405d234..19f3676 100644 --- a/pylintrc +++ b/pylintrc @@ -291,8 +291,7 @@ min-similarity-lines=4 # List of note tags to take in consideration, separated by a comma. notes=FIXME, - XXX, - TODO + XXX [TYPECHECK] diff --git a/pyms/config/confile.py b/pyms/config/confile.py index fb71d97..d30a46f 100644 --- a/pyms/config/confile.py +++ b/pyms/config/confile.py @@ -133,7 +133,7 @@ def __setattr__(self, name, value, *args, **kwargs): @staticmethod def __get_updated_configmap_file_env() -> str: + result = CONFIGMAP_FILE_ENVIRONMENT if (os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is not None) & (os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is None): return CONFIGMAP_FILE_ENVIRONMENT_LEGACY - else: - return CONFIGMAP_FILE_ENVIRONMENT + return result diff --git a/pyms/crypt/fernet.py b/pyms/crypt/fernet.py index 6a466f5..9e95864 100644 --- a/pyms/crypt/fernet.py +++ b/pyms/crypt/fernet.py @@ -64,7 +64,7 @@ def delete_key(self): @staticmethod def __get_updated_crypt_file_key_env() -> str: - if (os.getenv(CRYPT_FILE_KEY_ENVIRONMENT_LEGACY) is not None) & (os.getenv(CRYPT_FILE_KEY_ENVIRONMENT) is None): - return CRYPT_FILE_KEY_ENVIRONMENT_LEGACY - else: - return CRYPT_FILE_KEY_ENVIRONMENT + result = CRYPT_FILE_KEY_ENVIRONMENT + if (os.getenv(CRYPT_FILE_KEY_ENVIRONMENT_LEGACY) is not None) and (os.getenv(CRYPT_FILE_KEY_ENVIRONMENT) is None): + result = CRYPT_FILE_KEY_ENVIRONMENT_LEGACY + return result diff --git a/pyms/utils/utils.py b/pyms/utils/utils.py index af98ae4..9fee53b 100644 --- a/pyms/utils/utils.py +++ b/pyms/utils/utils.py @@ -36,9 +36,7 @@ def check_package_exists(package_name: Text) -> Union[Exception, bool]: def colored_text(msg, color: Colors, bold=False): - BOLD = "\033[1m" - RESET_ALL = "\033[0m" + result = "{}{}{}".format(color, msg, "\033[0m") if bold: - return "{}{}{}{}".format(BOLD, color, msg, RESET_ALL) - else: - return "{}{}{}".format(color, msg, RESET_ALL) + result = "{}{}".format("\033[1m", result) + return result From 579084400b521224fc75bfaf2fda0e61f4b867bc Mon Sep 17 00:00:00 2001 From: Rapejim Date: Mon, 12 Oct 2020 19:23:17 +0200 Subject: [PATCH 09/11] test: added test for better coverage --- pyms/config/confile.py | 2 +- tests/config-tests-debug-off.yml | 21 ++++++++++++++++ tests/config-tests-debug.yml | 21 ++++++++++++++++ tests/config-tests-deprecated.yml | 21 ++++++++++++++++ tests/test_config.py | 40 ++++++++++++++++++++++++++++++- 5 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 tests/config-tests-debug-off.yml create mode 100644 tests/config-tests-debug.yml create mode 100644 tests/config-tests-deprecated.yml diff --git a/pyms/config/confile.py b/pyms/config/confile.py index d30a46f..4ffeb93 100644 --- a/pyms/config/confile.py +++ b/pyms/config/confile.py @@ -135,5 +135,5 @@ def __setattr__(self, name, value, *args, **kwargs): def __get_updated_configmap_file_env() -> str: result = CONFIGMAP_FILE_ENVIRONMENT if (os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is not None) & (os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is None): - return CONFIGMAP_FILE_ENVIRONMENT_LEGACY + result = CONFIGMAP_FILE_ENVIRONMENT_LEGACY return result diff --git a/tests/config-tests-debug-off.yml b/tests/config-tests-debug-off.yml new file mode 100644 index 0000000..f3bf8d1 --- /dev/null +++ b/tests/config-tests-debug-off.yml @@ -0,0 +1,21 @@ +pyms: + services: + metrics: true + requests: + data: data + swagger: + path: "" + file: "swagger.yaml" + tracer: + client: "jaeger" + host: "localhost" + component_name: "Python Microservice" + config: + DEBUG: false + testing: true + app_name: "Python Microservice" + test_var: general + subservice1: + test: input + subservice2: + test: output diff --git a/tests/config-tests-debug.yml b/tests/config-tests-debug.yml new file mode 100644 index 0000000..13f6f16 --- /dev/null +++ b/tests/config-tests-debug.yml @@ -0,0 +1,21 @@ +pyms: + services: + metrics: true + requests: + data: data + swagger: + path: "" + file: "swagger.yaml" + tracer: + client: "jaeger" + host: "localhost" + component_name: "Python Microservice" + config: + DEBUG: true + testing: true + app_name: "Python Microservice" + test_var: general + subservice1: + test: input + subservice2: + test: output diff --git a/tests/config-tests-deprecated.yml b/tests/config-tests-deprecated.yml new file mode 100644 index 0000000..7144433 --- /dev/null +++ b/tests/config-tests-deprecated.yml @@ -0,0 +1,21 @@ +pyms: + services: + metrics: true + requests: + data: data + swagger: + path: "" + file: "swagger.yaml" + tracer: + client: "jaeger" + host: "localhost" + component_name: "Python Microservice" + config: + DEBUG: false + testing: true + app_name: "Python Microservice" + test_var: deprecated + subservice1: + test: input + subservice2: + test: output diff --git a/tests/test_config.py b/tests/test_config.py index c27c53a..66829b6 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,7 +5,8 @@ from pyms.config import get_conf, ConfFile from pyms.config.conf import validate_conf -from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT, LOGGER_NAME, CONFIG_BASE +from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT, CONFIGMAP_FILE_ENVIRONMENT_LEGACY, LOGGER_NAME, CONFIG_BASE, \ + CRYPT_FILE_KEY_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY from pyms.exceptions import AttrDoesNotExistException, ConfigDoesNotFoundException, ServiceDoesNotExistException, \ ConfigErrorException @@ -141,6 +142,16 @@ def test_without_params(self, mock_confile): class ConfValidateTests(unittest.TestCase): BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + def tearDown(self): + if os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is not None: + del os.environ[CONFIGMAP_FILE_ENVIRONMENT] + if os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is not None: + del os.environ[CONFIGMAP_FILE_ENVIRONMENT_LEGACY] + if os.getenv(CRYPT_FILE_KEY_ENVIRONMENT) is not None: + del os.environ[CRYPT_FILE_KEY_ENVIRONMENT] + if os.getenv(CRYPT_FILE_KEY_ENVIRONMENT_LEGACY) is not None: + del os.environ[CRYPT_FILE_KEY_ENVIRONMENT_LEGACY] + def test_wrong_block_no_pyms(self): with self.assertRaises(ConfigErrorException): validate_conf(path=os.path.join(self.BASE_DIR, "config-tests-bad-structure.yml")) @@ -152,3 +163,30 @@ def test_wrong_block_no_config(self): def test_wrong_block_not_valid_structure(self): with self.assertRaises(ConfigErrorException): validate_conf(path=os.path.join(self.BASE_DIR, "config-tests-bad-structure3.yml")) + + def test_validate_conf_file_with_env_deprecated(self): + self.all_validate_conf_combinations(True) + + def test_validate_conf_file_with_env_deprecated_and_updated(self): + os.environ[CONFIGMAP_FILE_ENVIRONMENT_LEGACY] = os.path.join(self.BASE_DIR, "config-tests-deprecated.yml") + self.all_validate_conf_combinations() + + def test_validate_crypt_file_with_env_deprecated(self): + os.environ[CRYPT_FILE_KEY_ENVIRONMENT_LEGACY] = os.path.join(self.BASE_DIR, "key.key") + self.all_validate_conf_combinations() + + def test_validate_crypt_file_with_env_deprecated_and_updated(self): + os.environ[CRYPT_FILE_KEY_ENVIRONMENT] = os.path.join(self.BASE_DIR, "key.key") + os.environ[CRYPT_FILE_KEY_ENVIRONMENT_LEGACY] = os.path.join(self.BASE_DIR, "key_deprecated.key") + self.all_validate_conf_combinations() + + def all_validate_conf_combinations(self, legacy=False): + config_env = CONFIGMAP_FILE_ENVIRONMENT + if legacy: + config_env = CONFIGMAP_FILE_ENVIRONMENT_LEGACY + os.environ[config_env] = os.path.join(self.BASE_DIR, "config-tests.yml") + validate_conf() + os.environ[config_env] = os.path.join(self.BASE_DIR, "config-tests-debug.yml") + validate_conf() + os.environ[config_env] = os.path.join(self.BASE_DIR, "config-tests-debug-off.yml") + validate_conf() From 94eca3ef41ec18b352e4736195eff67fc573f73c Mon Sep 17 00:00:00 2001 From: Rapejim Date: Mon, 12 Oct 2020 19:27:07 +0200 Subject: [PATCH 10/11] refactor: changed bitwise operators "&" to "and" --- pyms/config/confile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyms/config/confile.py b/pyms/config/confile.py index 4ffeb93..9036297 100644 --- a/pyms/config/confile.py +++ b/pyms/config/confile.py @@ -134,6 +134,6 @@ def __setattr__(self, name, value, *args, **kwargs): @staticmethod def __get_updated_configmap_file_env() -> str: result = CONFIGMAP_FILE_ENVIRONMENT - if (os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is not None) & (os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is None): + if (os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is not None) and (os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is None): result = CONFIGMAP_FILE_ENVIRONMENT_LEGACY return result From 1dc98e93b566b598ab38729d3a676b4ae0533dfc Mon Sep 17 00:00:00 2001 From: Rapejim Date: Mon, 12 Oct 2020 19:41:06 +0200 Subject: [PATCH 11/11] test: added test for better coverage --- tests/test_crypt.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/test_crypt.py b/tests/test_crypt.py index d4dcacf..8c4ebe7 100644 --- a/tests/test_crypt.py +++ b/tests/test_crypt.py @@ -7,7 +7,8 @@ from pyms.cloud.aws.kms import Crypt as CryptAws from pyms.config import get_conf -from pyms.constants import LOGGER_NAME, CONFIGMAP_FILE_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT, CONFIG_BASE +from pyms.constants import LOGGER_NAME, CONFIGMAP_FILE_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT, CONFIG_BASE, \ + CRYPT_FILE_KEY_ENVIRONMENT_LEGACY from pyms.crypt.driver import CryptAbstract, CryptResource from pyms.crypt.fernet import Crypt as CryptFernet from pyms.exceptions import FileDoesNotExistException @@ -78,7 +79,10 @@ def setUp(self): def tearDown(self): del os.environ[CONFIGMAP_FILE_ENVIRONMENT] - del os.environ[CRYPT_FILE_KEY_ENVIRONMENT] + if os.getenv(CRYPT_FILE_KEY_ENVIRONMENT): + del os.environ[CRYPT_FILE_KEY_ENVIRONMENT] + if os.getenv(CRYPT_FILE_KEY_ENVIRONMENT_LEGACY): + del os.environ[CRYPT_FILE_KEY_ENVIRONMENT_LEGACY] def test_encrypt_conf(self): crypt = CryptFernet(path=self.BASE_DIR) @@ -87,6 +91,15 @@ def test_encrypt_conf(self): crypt.delete_key() assert config.database_url == "http://database-url" + def test_encrypt_conf_deprecated_env(self): + os.environ[CRYPT_FILE_KEY_ENVIRONMENT_LEGACY] = os.getenv(CRYPT_FILE_KEY_ENVIRONMENT) + del os.environ[CRYPT_FILE_KEY_ENVIRONMENT] + crypt = CryptFernet(path=self.BASE_DIR) + crypt._loader.put_file(b"9IXx2F5d5Ob-h5xdCnFSUXhuFKLGRibvLfSbixpcfCw=", "wb") + config = get_conf(service=CONFIG_BASE, uppercase=True, crypt=CryptFernet) + crypt.delete_key() + assert config.database_url == "http://database-url" + class GetConfigEncryptedAWS(unittest.TestCase): BASE_DIR = os.path.dirname(os.path.abspath(__file__))