From a2a86efd7b74ea3cb3dc084ad12004754a5aa932 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Sun, 16 Feb 2020 21:20:10 +0100 Subject: [PATCH 01/23] Added crypt options --- Pipfile | 1 + Pipfile.lock | 186 +++++++++++++++++++++---------- pyms/cmd/__init__.py | 1 + pyms/cmd/main.py | 96 ++++++++++++++++ pyms/config/confile.py | 50 +++------ pyms/constants.py | 6 + pyms/utils/crypt.py | 57 ++++++++++ pyms/utils/files.py | 57 ++++++++++ setup.py | 5 + tests/config-tests-encrypted.yml | 23 ++++ tests/test_cmd.py | 40 +++++++ tests/test_config.py | 36 +++--- tests/test_utils.py | 23 +++- 13 files changed, 472 insertions(+), 109 deletions(-) create mode 100755 pyms/cmd/__init__.py create mode 100755 pyms/cmd/main.py create mode 100644 pyms/utils/crypt.py create mode 100644 pyms/utils/files.py create mode 100644 tests/config-tests-encrypted.yml create mode 100644 tests/test_cmd.py diff --git a/Pipfile b/Pipfile index d83823d..1d5a350 100644 --- a/Pipfile +++ b/Pipfile @@ -15,6 +15,7 @@ flask-opentracing = "*" opentracing = ">=2.1" opentracing-instrumentation = "==3.2.1" prometheus_client = ">=0.7.1" +cryptography = "*" [dev-packages] requests-mock = "*" diff --git a/Pipfile.lock b/Pipfile.lock index e87c56b..8885dd3 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "2f99e1d08fceb712f6fdb9395e19ce6e2cee3ee17bd71fdf91c40adb557c1ebb" + "sha256": "202ec3deff30d8c4b65ba37d8244ef5b350bd9d0508697e84ce4ed61107d1820" }, "pipfile-spec": 6, "requires": { @@ -37,6 +37,39 @@ ], "version": "==2019.11.28" }, + "cffi": { + "hashes": [ + "sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff", + "sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b", + "sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac", + "sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0", + "sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384", + "sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26", + "sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6", + "sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b", + "sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e", + "sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd", + "sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2", + "sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66", + "sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc", + "sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8", + "sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55", + "sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4", + "sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5", + "sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d", + "sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78", + "sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa", + "sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793", + "sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f", + "sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a", + "sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f", + "sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30", + "sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f", + "sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3", + "sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c" + ], + "version": "==1.14.0" + }, "chardet": { "hashes": [ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", @@ -76,6 +109,33 @@ ], "version": "==0.6.0.post1" }, + "cryptography": { + "hashes": [ + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + ], + "index": "pypi", + "version": "==2.8" + }, "flask": { "hashes": [ "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", @@ -134,10 +194,10 @@ }, "jinja2": { "hashes": [ - "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250", - "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49" + "sha256:c10142f819c2d22bdcd17548c46fa9b77cf4fda45097854c689666bf425e7484", + "sha256:c922560ac46888d47384de1dbdc3daaa2ea993af4b26a436dec31fa2c19ec668" ], - "version": "==2.11.1" + "version": "==3.0.0a1" }, "jsonschema": { "hashes": [ @@ -214,6 +274,12 @@ "index": "pypi", "version": "==0.7.1" }, + "pycparser": { + "hashes": [ + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + ], + "version": "==2.19" + }, "pyrsistent": { "hashes": [ "sha256:cdc7b5e3ed77bed61270a47d35434a30617b9becdf2478af76ad2c6ade307280" @@ -301,10 +367,10 @@ }, "werkzeug": { "hashes": [ - "sha256:7f91416b2e6902e39190552d5e05bc94c0d5034688753fceb28eaa9d16795027", - "sha256:834e6f4828864350b7c173a2be027f0a692512bbe9d7ddd52da8e29ff991a726" + "sha256:169ba8a33788476292d04186ab33b01d6add475033dfc07215e6d219cc077096", + "sha256:6dc65cf9091cf750012f56f2cad759fa9e879f511b5ff8685e456b4e3bf90d16" ], - "version": "==1.0.0rc1" + "version": "==1.0.0" }, "wrapt": { "hashes": [ @@ -314,10 +380,10 @@ }, "zipp": { "hashes": [ - "sha256:ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28", - "sha256:feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98" + "sha256:5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50", + "sha256:d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e" ], - "version": "==2.1.0" + "version": "==2.2.0" } }, "develop": { @@ -415,6 +481,12 @@ "index": "pypi", "version": "==4.5.4" }, + "distlib": { + "hashes": [ + "sha256:2e166e231a26b36d6dfe35a48c4464346620f8645ed0ace01ee31822b288de21" + ], + "version": "==0.3.0" + }, "entrypoints": { "hashes": [ "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", @@ -439,17 +511,17 @@ }, "gitdb2": { "hashes": [ - "sha256:1b6df1433567a51a4a9c1a5a0de977aa351a405cc56d7d35f3388bad1f630350", - "sha256:96bbb507d765a7f51eb802554a9cfe194a174582f772e0d89f4e87288c288b7b" + "sha256:0a84c85d07166cefaf10fa4b728a8a1b847a7f0d081e2fbe083185462158c126", + "sha256:51380b9d8b977b42ac9fc3a8e17af90a56ffc573ca9c6b79f99033b04f29f457" ], - "version": "==2.0.6" + "version": "==3.0.0" }, "gitpython": { "hashes": [ - "sha256:9c2398ffc3dcb3c40b27324b316f08a4f93ad646d5a6328cafbb871aa79f5e42", - "sha256:c155c6a2653593ccb300462f6ef533583a913e17857cfef8fc617c246b6dc245" + "sha256:99c77677f31f255e130f3fed4c8e0eebb35f1a09df98ff965fff6774f71688cf", + "sha256:99cd0403cecd8a13b95d2e045b9fcaa7837137fcc5ec3105f2c413305d82c143" ], - "version": "==3.0.5" + "version": "==3.0.7" }, "googleapis-common-protos": { "hashes": [ @@ -489,17 +561,17 @@ }, "jinja2": { "hashes": [ - "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250", - "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49" + "sha256:c10142f819c2d22bdcd17548c46fa9b77cf4fda45097854c689666bf425e7484", + "sha256:c922560ac46888d47384de1dbdc3daaa2ea993af4b26a436dec31fa2c19ec668" ], - "version": "==2.11.1" + "version": "==3.0.0a1" }, "jsonpickle": { "hashes": [ - "sha256:d0c5a4e6cb4e58f6d5406bdded44365c2bcf9c836c4f52910cc9ba7245a59dc2", - "sha256:d3e922d781b1d0096df2dad89a2e1f47177d7969b596aea806a9d91b4626b29b" + "sha256:71bca2b80ae28af4e3f86629ef247100af7f97032b5ca8d791c1f8725b411d95", + "sha256:efc6839cb341985f0c24f98650a4c1063a2877c236ffd3d7e1662f0c482bac93" ], - "version": "==1.2" + "version": "==1.3" }, "lazy-object-proxy": { "hashes": [ @@ -544,10 +616,10 @@ }, "markdown": { "hashes": [ - "sha256:2e50876bcdd74517e7b71f3e7a76102050edec255b3983403f1a63e7c8a41e7a", - "sha256:56a46ac655704b91e5b7e6326ce43d5ef72411376588afa1dd90e881b83c7e8c" + "sha256:90fee683eeabe1a92e149f7ba74e5ccdc81cd397bd6c516d93a8da0ef90b6902", + "sha256:e4795399163109457d4c5af2183fbe6b60326c17cfdf25ce6e7474c6624f725d" ], - "version": "==3.1.1" + "version": "==3.2.1" }, "markupsafe": { "hashes": [ @@ -639,26 +711,26 @@ }, "protobuf": { "hashes": [ - "sha256:0329e86a397db2a83f9dcbe21d9be55a47f963cdabc893c3a24f4d3a8f117c37", - "sha256:0a7219254afec0d488211f3d482d8ed57e80ae735394e584a98d8f30a8c88a36", - "sha256:14d6ac53df9cb5bb87c4f91b677c1bc5cec9c0fd44327f367a3c9562de2877c4", - "sha256:180fc364b42907a1d2afa183ccbeffafe659378c236b1ec3daca524950bb918d", - "sha256:3d7a7d8d20b4e7a8f63f62de2d192cfd8b7a53c56caba7ece95367ca2b80c574", - "sha256:3f509f7e50d806a434fe4a5fbf602516002a0f092889209fff7db82060efffc0", - "sha256:4571da974019849201fc1ec6626b9cea54bd11b6bed140f8f737c0a33ea37de5", - "sha256:56bd1d84fbf4505c7b73f04de987eef5682e5752c811141b0186a3809bfb396f", - "sha256:680c668d00b5eff08b86aef9e5ba9a705e621ea05d39071cfea8e28cb2400946", - "sha256:6b5b947dc8b3f2aec0eaad65b0b5113fcd642c358c31357c647da6281ee31104", - "sha256:6e96dffaf4d0a9a329e528b353ba62fd9ef13599688723d96bc9c165d0b6871e", - "sha256:919f0d6f6addc836d08658eba3b52be2e92fd3e76da3ce00c325d8e9826d17c7", - "sha256:9c7b19c30cf0644afd0e4218b13f637ce54382fdcb1c8f75bf3e84e49a5f6d0a", - "sha256:a2e6f57114933882ec701807f217df2fb4588d47f71f227c0a163446b930d507", - "sha256:a6b970a2eccfcbabe1acf230fbf112face1c4700036c95e195f3554d7bcb04c1", - "sha256:bc45641cbcdea068b67438244c926f9fd3e5cbdd824448a4a64370610df7c593", - "sha256:d61b14a9090da77fe87e38ba4c6c43d3533dcbeb5d84f5474e7ac63c532dcc9c", - "sha256:d6faf5dbefb593e127463f58076b62fcfe0784187be8fe1aa9167388f24a22a1" - ], - "version": "==3.11.2" + "sha256:0bae429443cc4748be2aadfdaf9633297cfaeb24a9a02d0ab15849175ce90fab", + "sha256:24e3b6ad259544d717902777b33966a1a069208c885576254c112663e6a5bb0f", + "sha256:310a7aca6e7f257510d0c750364774034272538d51796ca31d42c3925d12a52a", + "sha256:52e586072612c1eec18e1174f8e3bb19d08f075fc2e3f91d3b16c919078469d0", + "sha256:73152776dc75f335c476d11d52ec6f0f6925774802cd48d6189f4d5d7fe753f4", + "sha256:7774bbbaac81d3ba86de646c39f154afc8156717972bf0450c9dbfa1dc8dbea2", + "sha256:82d7ac987715d8d1eb4068bf997f3053468e0ce0287e2729c30601feb6602fee", + "sha256:8eb9c93798b904f141d9de36a0ba9f9b73cc382869e67c9e642c0aba53b0fc07", + "sha256:adf0e4d57b33881d0c63bb11e7f9038f98ee0c3e334c221f0858f826e8fb0151", + "sha256:c40973a0aee65422d8cb4e7d7cbded95dfeee0199caab54d5ab25b63bce8135a", + "sha256:c77c974d1dadf246d789f6dad1c24426137c9091e930dbf50e0a29c1fcf00b1f", + "sha256:dd9aa4401c36785ea1b6fff0552c674bdd1b641319cb07ed1fe2392388e9b0d7", + "sha256:e11df1ac6905e81b815ab6fd518e79be0a58b5dc427a2cf7208980f30694b956", + "sha256:e2f8a75261c26b2f5f3442b0525d50fd79a71aeca04b5ec270fc123536188306", + "sha256:e512b7f3a4dd780f59f1bf22c302740e27b10b5c97e858a6061772668cd6f961", + "sha256:ef2c2e56aaf9ee914d3dccc3408d42661aaf7d9bb78eaa8f17b2e6282f214481", + "sha256:fac513a9dc2a74b99abd2e17109b53945e364649ca03d9f7a0b96aa8d1807d0a", + "sha256:fdfb6ad138dbbf92b5dbea3576d7c8ba7463173f7d2cb0ca1bd336ec88ddbd80" + ], + "version": "==3.11.3" }, "py": { "hashes": [ @@ -760,10 +832,10 @@ }, "stevedore": { "hashes": [ - "sha256:01d9f4beecf0fbd070ddb18e5efb10567801ba7ef3ddab0074f54e3cd4e91730", - "sha256:e0739f9739a681c7a1fda76a102b65295e96a144ccdb552f2ae03c5f0abe8a14" + "sha256:18afaf1d623af5950cc0f7e75e70f917784c73b652a34a12d90b309451b5500b", + "sha256:a4e7dc759fb0f2e3e2f7d8ffe2358c19d45b9b8297f393ef1256858d82f69c9b" ], - "version": "==1.31.0" + "version": "==1.32.0" }, "thrift": { "hashes": [ @@ -792,11 +864,11 @@ }, "tox": { "hashes": [ - "sha256:06ba73b149bf838d5cd25dc30c2dd2671ae5b2757cf98e5c41a35fe449f131b3", - "sha256:806d0a9217584558cc93747a945a9d9bff10b141a5287f0c8429a08828a22192" + "sha256:5c45d08f1dcc9bc97cdea3e5a69c8f4ad042cc37cbe6cf53e126f4a9005b7d3b", + "sha256:73e2ade68cd71a2765ee739ecc27c8c92e9b9c09acbad0f2c5307b20214e0d13" ], "index": "pypi", - "version": "==3.14.3" + "version": "==3.14.4" }, "typed-ast": { "hashes": [ @@ -834,10 +906,10 @@ }, "virtualenv": { "hashes": [ - "sha256:916497082376027a387c49af092a4316c0e9db753f95ea1af22eba569f107cfe", - "sha256:9a87270123622593ad454a81055b771a8898590ff3d6f3abbe48c4111ff49e79" + "sha256:08f3623597ce73b85d6854fb26608a6f39ee9d055c81178dc6583803797f8994", + "sha256:de2cbdd5926c48d7b84e0300dea9e8f276f61d186e8e49223d71d91250fbaebd" ], - "version": "==20.0.0b1" + "version": "==20.0.4" }, "wcwidth": { "hashes": [ @@ -854,10 +926,10 @@ }, "zipp": { "hashes": [ - "sha256:ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28", - "sha256:feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98" + "sha256:5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50", + "sha256:d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e" ], - "version": "==2.1.0" + "version": "==2.2.0" } } } diff --git a/pyms/cmd/__init__.py b/pyms/cmd/__init__.py new file mode 100755 index 0000000..82b6bf2 --- /dev/null +++ b/pyms/cmd/__init__.py @@ -0,0 +1 @@ +from pyms.cmd.main import Command diff --git a/pyms/cmd/main.py b/pyms/cmd/main.py new file mode 100755 index 0000000..b29f413 --- /dev/null +++ b/pyms/cmd/main.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function + +import argparse +import sys + +from pyms.utils.crypt import Crypt + + +class Command(object): + config = None + + parser = None + + args = [] + + def __init__(self, *args, **kwargs): + arguments = kwargs.get("arguments", False) + autorun = kwargs.get("autorun", True) + if not arguments: + arguments = sys.argv[1:] + + parser = argparse.ArgumentParser(description='Python Microservices') + + commands = parser.add_subparsers(title="Commands", description='Available commands', dest='command_name') + + parser_encrypt = commands.add_parser('encrypt', help='Encrypt a string') + parser_encrypt.add_argument("encrypt", default='', type=str, help='Encrypt a string') + + parser_create_key = commands.add_parser('create-key', help='Encrypt a string') + parser_create_key.add_argument("create_key", action='store_true', + help='Generate a Key to encrypt strings in config') + + parser.add_argument("-v", "--verbose", default="", type=str, help="Verbose ") + + args = parser.parse_args(arguments) + try: + self.create_key = args.create_key + except AttributeError: + self.create_key = False + try: + self.encrypt = args.encrypt + except AttributeError: + self.encrypt = "" + self.verbose = len(args.verbose) + if autorun: + result = self.run() + if result: + self.exit_ok("OK") + else: + self.print_error("ERROR") + + def get_input(self, msg): + return input(msg) + + def run(self): + crypt = Crypt() + if self.create_key: + path = crypt._loader.get_or_setpath() + pwd = self.get_input('Type a password to generate the key file: ') + generate_file = self.get_input('Do you want to generate a file in ? [Y/n]'.format(path)) + generate_file = generate_file.lower() != "n" + key = crypt.generate_key(pwd, generate_file) + if generate_file: + self.print_ok("File {} generated OK".format(path)) + else: + self.print_ok("Key generated: {}".format(key)) + if self.encrypt: + encrypted = crypt.encrypt(self.encrypt) + self.print_ok("Encrypted OK: {}".format(encrypted)) + + @staticmethod + def print_ok(msg=""): + print('\033[92m\033[1m ' + msg + ' \033[0m\033[0m') + + def print_verbose(self, msg=""): + if self.verbose: + print(msg) + + @staticmethod + def print_error(msg=""): + print('\033[91m\033[1m ' + msg + ' \033[0m\033[0m') + + def exit_with_error(self, msg=""): + self.print_error(msg) + sys.exit(2) + + def exit_ok(self, msg=""): + self.print_ok(msg) + sys.exit(0) + + +if __name__ == '__main__': # pragma: no cover + cmd = Command(arguments=sys.argv[1:], autorun=False) + cmd.run() diff --git a/pyms/config/confile.py b/pyms/config/confile.py index 49d5771..e314828 100644 --- a/pyms/config/confile.py +++ b/pyms/config/confile.py @@ -1,27 +1,23 @@ """Module to read yaml or json conf""" import logging -import os - +import re import anyconfig -from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT, LOGGER_NAME +from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT, LOGGER_NAME, DEFAULT_CONFIGMAP_FILENAME from pyms.exceptions import AttrDoesNotExistException, ConfigDoesNotFoundException +from pyms.utils.crypt import Crypt +from pyms.utils.files import LoadFile logger = logging.getLogger(LOGGER_NAME) -config_cache = {} - class ConfFile(dict): """Recursive get configuration from dictionary, a config file in JSON or YAML format from a path or `CONFIGMAP_FILE` environment variable. **Atributes:** * empty_init: Allow blank variables - * default_file: search for config.yml file """ _empty_init = False - _default_file = "config.yml" - __path = None def __init__(self, *args, **kwargs): """ @@ -34,12 +30,12 @@ def __init__(self, *args, **kwargs): self[key] = getattr(obj, key) ``` """ + self._loader = LoadFile(kwargs.get("path"), CONFIGMAP_FILE_ENVIRONMENT, DEFAULT_CONFIGMAP_FILENAME) + self._crypt = Crypt(path=kwargs.get("path")) self._empty_init = kwargs.get("empty_init", False) config = kwargs.get("config") if config is None: - self.set_path(kwargs.get("path")) - config = self._get_conf_from_file() or self._get_conf_from_env() - + config = self._loader.get_file(anyconfig.load) if not config: if self._empty_init: config = {} @@ -50,16 +46,17 @@ def __init__(self, *args, **kwargs): super(ConfFile, self).__init__(config) - def set_path(self, path): - self.__path = path - def to_flask(self): return ConfFile(config={k.upper(): v for k, v in self.items()}) def set_config(self, config): config = dict(self.normalize_config(config)) for k, v in config.items(): - setattr(self, k, v) + if k.lower().startswith("enc_"): + k_not_crypt = re.compile(re.escape('enc_'), re.IGNORECASE) + setattr(self, k_not_crypt.sub('', k), self._crypt.decrypt(v)) + else: + setattr(self, k, v) return config def normalize_config(self, config): @@ -91,28 +88,9 @@ def __getattr__(self, name, *args, **kwargs): return ConfFile(config={}, empty_init=self._empty_init) raise AttrDoesNotExistException("Variable {} not exist in the config file".format(name)) - def _get_conf_from_env(self): - config_file = os.environ.get(CONFIGMAP_FILE_ENVIRONMENT, self._default_file) - logger.debug("[CONF] Searching file in ENV[{}]: {}...".format(CONFIGMAP_FILE_ENVIRONMENT, config_file)) - self.set_path(config_file) - return self._get_conf_from_file() - - def _get_conf_from_file(self) -> dict: - if not self.__path or not os.path.isfile(self.__path): - logger.debug("[CONF] Configmap {} NOT FOUND".format(self.__path)) - return {} - if self.__path not in config_cache: - logger.debug("[CONF] Configmap {} found".format(self.__path)) - config_cache[self.__path] = anyconfig.load(self.__path) - return config_cache[self.__path] - - def load(self): - config_src = self._get_conf_from_file() or self._get_conf_from_env() - self.set_config(config_src) - def reload(self): - config_cache.pop(self.__path, None) - self.load() + config_src = self._loader.reload(anyconfig.load) + self.set_config(config_src) def __setattr__(self, name, value, *args, **kwargs): super(ConfFile, self).__setattr__(name, value) diff --git a/pyms/constants.py b/pyms/constants.py index bf5a0c7..de335d3 100644 --- a/pyms/constants.py +++ b/pyms/constants.py @@ -1,5 +1,11 @@ CONFIGMAP_FILE_ENVIRONMENT = "CONFIGMAP_FILE" +DEFAULT_CONFIGMAP_FILENAME = "config.yml" + +CRYPT_FILE_KEY_ENVIRONMENT = "KEY_FILE" + +DEFAULT_KEY_FILENAME = "key.key" + LOGGER_NAME = "pyms" CONFIG_BASE = "pyms.config" diff --git a/pyms/utils/crypt.py b/pyms/utils/crypt.py new file mode 100644 index 0000000..e18968b --- /dev/null +++ b/pyms/utils/crypt.py @@ -0,0 +1,57 @@ +import base64 +import os +from typing import Text + +from cryptography.fernet import Fernet +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + +from pyms.constants import CRYPT_FILE_KEY_ENVIRONMENT, DEFAULT_KEY_FILENAME +from pyms.exceptions import FileDoesNotExistException +from pyms.utils.files import LoadFile + + +class Crypt: + def __init__(self, *args, **kwargs): + self._loader = LoadFile(kwargs.get("path"), CRYPT_FILE_KEY_ENVIRONMENT, DEFAULT_KEY_FILENAME) + + def generate_key(self, password: Text, write_to_file: bool = False): + password = password.encode() # Convert to type bytes + salt = os.urandom(16) + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), + length=32, + salt=salt, + iterations=100000, + backend=default_backend() + ) + key = base64.urlsafe_b64encode(kdf.derive(password)) # Can only use kdf once + if write_to_file: + self._loader.put_file(key, 'wb') + return key + + def read_key(self): + key = self._loader.get_file() + if not key: + raise FileDoesNotExistException( + "Decrypt key {} not exists. You must set a correct env var {} " + "or run `pyms crypt create-key` command".format(self._loader.path, CRYPT_FILE_KEY_ENVIRONMENT)) + return key + + def encrypt(self, message): + key = self.read_key() + message = message.encode() + f = Fernet(key) + encrypted = f.encrypt(message) + return encrypted + + def decrypt(self, encrypted): + key = self.read_key() + encrypted = encrypted.encode() + f = Fernet(key) + decrypted = f.decrypt(encrypted) + return str(decrypted, encoding="utf-8") + + def delete_key(self): + os.remove(self._loader.get_or_setpath()) diff --git a/pyms/utils/files.py b/pyms/utils/files.py new file mode 100644 index 0000000..c1530c8 --- /dev/null +++ b/pyms/utils/files.py @@ -0,0 +1,57 @@ +import logging +import os + +from pyms.constants import LOGGER_NAME + +files_cached = {} + +logger = logging.getLogger(LOGGER_NAME) + + +class LoadFile: + default_file = None + file_env_location = None + path = None + + def __init__(self, path, env_var, default_filename): + self.path = path + self.file_env_location = env_var + self.default_file = default_filename + + def get_file(self, fn=None): + return self._get_conf_from_file(fn) or self._get_conf_from_env(fn) + + def put_file(self, content, mode="w"): + self.get_or_setpath() + file_to_write = open(self.path, mode) + file_to_write.write(content) # The key is type bytes still + file_to_write.close() + + def get_or_setpath(self): + config_file = os.environ.get(self.file_env_location, self.default_file) + logger.debug("Searching file in ENV[{}]: {}...".format(self.file_env_location, config_file)) + self.path = config_file + return self.path + + def _get_conf_from_env(self, fn=None): + self.get_or_setpath() + return self._get_conf_from_file(fn) + + def _get_conf_from_file(self, fn=None): + if not self.path or not os.path.isfile(self.path): + logger.debug("File {} NOT FOUND".format(self.path)) + return {} + if self.path not in files_cached: + logger.debug("[CONF] Configmap {} found".format(self.path)) + if fn: + files_cached[self.path] = fn(self.path) + else: + file_to_read = open(self.path, 'rb') + content = file_to_read.read() # The key will be type bytes + file_to_read.close() + files_cached[self.path] = content + return files_cached[self.path] + + def reload(self, fn=None): + files_cached.pop(self.path, None) + return self.get_file(fn) diff --git a/setup.py b/setup.py index f598994..78ae85c 100644 --- a/setup.py +++ b/setup.py @@ -56,5 +56,10 @@ install_requires=install_requires, tests_require=tests_require, include_package_data=True, + entry_points={ + 'console_scripts': [ + 'pyms = pyms.cmd.main:Command' + ] + }, zip_safe=True, ) diff --git a/tests/config-tests-encrypted.yml b/tests/config-tests-encrypted.yml new file mode 100644 index 0000000..46ecaee --- /dev/null +++ b/tests/config-tests-encrypted.yml @@ -0,0 +1,23 @@ +pyms: + services: + metrics: true + requests: + data: data + swagger: + path: "" + file: "swagger.yaml" + tracer: + client: "jaeger" + host: "localhost" + component_name: "Python Microservice" + config: + debug: true + testing: true + app_name: "Python Microservice" + application_root: / + test_var: general + subservice1: + test: input + subservice2: + test: output + enc_database_url: gAAAAABeSZ714r99iRIxhoH77vTdRJ0iqSymShfqgGN9PJveqhQWmshRDuV2a8sATey8_lHkln0TwezczucH-aJHGP_LyEiPxwM-88clNa7FB1u4g7Iaw3A= diff --git a/tests/test_cmd.py b/tests/test_cmd.py new file mode 100644 index 0000000..120dc09 --- /dev/null +++ b/tests/test_cmd.py @@ -0,0 +1,40 @@ +"""Test common rest operations wrapper. +""" +import os +import unittest +from unittest.mock import patch + +import pytest + +from pyms.cmd import Command +from pyms.exceptions import FileDoesNotExistException +from pyms.utils.crypt import Crypt + + +class TestCmd(unittest.TestCase): + BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + + def test_crypt_file_error(self): + arguments = ["encrypt", "prueba"] + cmd = Command(arguments=arguments, autorun=False) + with pytest.raises(FileDoesNotExistException) as excinfo: + cmd.run() + assert ("Decrypt key key.key not exists. You must set a correct env var KEY_FILE or run " + "`pyms crypt create-key` command") \ + in str(excinfo.value) + + def test_crypt_file_ok(self): + crypt = Crypt() + crypt.generate_key("mypassword", True) + arguments = ["encrypt", "prueba"] + cmd = Command(arguments=arguments, autorun=False) + cmd.run() + crypt.delete_key() + + @patch('pyms.cmd.main.Command.get_input', return_value='Y') + def test_generate_file_ok(self, input): + crypt = Crypt() + arguments = ["create-key", ] + cmd = Command(arguments=arguments, autorun=False) + cmd.run() + crypt.delete_key() diff --git a/tests/test_config.py b/tests/test_config.py index e97519a..c7c746d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -4,8 +4,9 @@ from unittest import mock from pyms.config import get_conf, ConfFile -from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT, LOGGER_NAME, CONFIG_BASE +from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT, LOGGER_NAME, CONFIG_BASE, CRYPT_FILE_KEY_ENVIRONMENT from pyms.exceptions import AttrDoesNotExistException, ConfigDoesNotFoundException, ServiceDoesNotExistException +from pyms.utils.crypt import Crypt logger = logging.getLogger(LOGGER_NAME) @@ -97,20 +98,6 @@ def test_example_test_json_file(self): self.assertEqual(config.pyms.config.test_var, "general") -class ConfCacheTests(unittest.TestCase): - BASE_DIR = os.path.dirname(os.path.abspath(__file__)) - def test_get_cache(self): - config = ConfFile(path=os.path.join(self.BASE_DIR, "config-tests-cache.yml")) - config.set_path(os.path.join(self.BASE_DIR, "config-tests-cache2.yml")) - self.assertEqual(config.pyms.config.my_cache, 1234) - - def test_get_cache_and_reload(self): - config = ConfFile(path=os.path.join(self.BASE_DIR, "config-tests-cache.yml")) - config.set_path(os.path.join(self.BASE_DIR, "config-tests-cache2.yml")) - config.reload() - self.assertEqual(config.pyms.config.my_cache, 12345678) - - class ConfNotExistTests(unittest.TestCase): def test_empty_conf(self): config = ConfFile(empty_init=True) @@ -148,3 +135,22 @@ def test_default_flask(self): def test_without_params(self, mock_confile): with self.assertRaises(ServiceDoesNotExistException): get_conf() + + +class GetConfigEncrypted(unittest.TestCase): + BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + + def setUp(self): + os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(self.BASE_DIR, "config-tests-encrypted.yml") + os.environ[CRYPT_FILE_KEY_ENVIRONMENT] = os.path.join(self.BASE_DIR, "key.key") + + def tearDown(self): + del os.environ[CONFIGMAP_FILE_ENVIRONMENT] + del os.environ[CRYPT_FILE_KEY_ENVIRONMENT] + + def test_encrypt_conf(self): + crypt = Crypt(path=self.BASE_DIR) + crypt._loader.put_file(b"9IXx2F5d5Ob-h5xdCnFSUXhuFKLGRibvLfSbixpcfCw=", "wb") + config = get_conf(service=CONFIG_BASE, uppercase=True) + crypt.delete_key() + assert config.database_url == "http://database-url" diff --git a/tests/test_utils.py b/tests/test_utils.py index 910e179..57d2a5f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,8 +5,9 @@ import pytest -from pyms.exceptions import PackageNotExists +from pyms.exceptions import PackageNotExists, FileDoesNotExistException from pyms.utils import check_package_exists, import_package +from pyms.utils.crypt import Crypt class ConfUtils(unittest.TestCase): @@ -21,3 +22,23 @@ def test_check_package_exists_exception(self): def test_import_package(self): os_import = import_package("os") assert os_import == os + + +class CryptUtils(unittest.TestCase): + BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + + def test_crypt_file_error(self): + crypt = Crypt() + with pytest.raises(FileDoesNotExistException) as excinfo: + crypt.read_key() + assert ("Decrypt key key.key not exists. You must set a correct env var KEY_FILE or run " + "`pyms crypt create-key` command") \ + in str(excinfo.value) + + def test_crypt_file_ok(self): + crypt = Crypt() + crypt.generate_key("mypassword", True) + message = "My crypt message" + encrypt_message = crypt.encrypt(message) + assert message == crypt.decrypt(str(encrypt_message, encoding="utf-8")) + crypt.delete_key() From 0d062f863b0c7281049e3a879a2538b7240e08cf Mon Sep 17 00:00:00 2001 From: avara1986 Date: Sun, 16 Feb 2020 21:37:23 +0100 Subject: [PATCH 02/23] Fix pylint and flake8 --- pyms/cmd/__init__.py | 2 ++ pyms/cmd/main.py | 12 +++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pyms/cmd/__init__.py b/pyms/cmd/__init__.py index 82b6bf2..81c17cc 100755 --- a/pyms/cmd/__init__.py +++ b/pyms/cmd/__init__.py @@ -1 +1,3 @@ from pyms.cmd.main import Command + +__all__ = ['Command'] diff --git a/pyms/cmd/main.py b/pyms/cmd/main.py index b29f413..989dcd8 100755 --- a/pyms/cmd/main.py +++ b/pyms/cmd/main.py @@ -8,7 +8,7 @@ from pyms.utils.crypt import Crypt -class Command(object): +class Command: config = None parser = None @@ -51,15 +51,16 @@ def __init__(self, *args, **kwargs): else: self.print_error("ERROR") - def get_input(self, msg): - return input(msg) + @staticmethod + def get_input(msg): + return input(msg) # nosec def run(self): crypt = Crypt() if self.create_key: - path = crypt._loader.get_or_setpath() + path = crypt._loader.get_or_setpath() # pylint: disable=protected-access pwd = self.get_input('Type a password to generate the key file: ') - generate_file = self.get_input('Do you want to generate a file in ? [Y/n]'.format(path)) + generate_file = self.get_input('Do you want to generate a file in {}? [Y/n]'.format(path)) generate_file = generate_file.lower() != "n" key = crypt.generate_key(pwd, generate_file) if generate_file: @@ -69,6 +70,7 @@ def run(self): if self.encrypt: encrypted = crypt.encrypt(self.encrypt) self.print_ok("Encrypted OK: {}".format(encrypted)) + return True @staticmethod def print_ok(msg=""): From 7b05354f7f3ab592686bf15502ad14c6249b8d71 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Sun, 16 Feb 2020 21:47:26 +0100 Subject: [PATCH 03/23] No cover to no testable conditions --- pyms/cmd/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyms/cmd/main.py b/pyms/cmd/main.py index 989dcd8..9257539 100755 --- a/pyms/cmd/main.py +++ b/pyms/cmd/main.py @@ -44,7 +44,7 @@ def __init__(self, *args, **kwargs): except AttributeError: self.encrypt = "" self.verbose = len(args.verbose) - if autorun: + if autorun: # pragma: no cover result = self.run() if result: self.exit_ok("OK") @@ -52,7 +52,7 @@ def __init__(self, *args, **kwargs): self.print_error("ERROR") @staticmethod - def get_input(msg): + def get_input(msg): # pragma: no cover return input(msg) # nosec def run(self): @@ -76,19 +76,19 @@ def run(self): def print_ok(msg=""): print('\033[92m\033[1m ' + msg + ' \033[0m\033[0m') - def print_verbose(self, msg=""): + def print_verbose(self, msg=""): # pragma: no cover if self.verbose: print(msg) @staticmethod - def print_error(msg=""): + def print_error(msg=""): # pragma: no cover print('\033[91m\033[1m ' + msg + ' \033[0m\033[0m') - def exit_with_error(self, msg=""): + def exit_with_error(self, msg=""): # pragma: no cover self.print_error(msg) sys.exit(2) - def exit_ok(self, msg=""): + def exit_ok(self, msg=""): # pragma: no cover self.print_ok(msg) sys.exit(0) From 26a8b9513f04fe9e3512f3f9cf72dd12898fada3 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Sun, 16 Feb 2020 22:18:06 +0100 Subject: [PATCH 04/23] No cover to no testable conditions --- pyms/cmd/main.py | 2 +- tests/test_cmd.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pyms/cmd/main.py b/pyms/cmd/main.py index 9257539..3c9f4a7 100755 --- a/pyms/cmd/main.py +++ b/pyms/cmd/main.py @@ -18,7 +18,7 @@ class Command: def __init__(self, *args, **kwargs): arguments = kwargs.get("arguments", False) autorun = kwargs.get("autorun", True) - if not arguments: + if not arguments: # pragma: no cover arguments = sys.argv[1:] parser = argparse.ArgumentParser(description='Python Microservices') diff --git a/tests/test_cmd.py b/tests/test_cmd.py index 120dc09..e9e7535 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -38,3 +38,13 @@ def test_generate_file_ok(self, input): cmd = Command(arguments=arguments, autorun=False) cmd.run() crypt.delete_key() + + @patch('pyms.cmd.main.Command.get_input', return_value='n') + def test_output_key(self, input): + crypt = Crypt() + arguments = ["create-key", ] + cmd = Command(arguments=arguments, autorun=False) + cmd.run() + with pytest.raises(FileNotFoundError) as excinfo: + crypt.delete_key() + assert ("[Errno 2] No such file or directory: 'key.key'") in str(excinfo.value) From 9114af6a62ad632fae2dcbdad745785e3546b4ef Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 17 Feb 2020 18:53:16 +0100 Subject: [PATCH 05/23] Removed encrypted var after decrypted --- pyms/config/confile.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/pyms/config/confile.py b/pyms/config/confile.py index eed876c..7f1df27 100644 --- a/pyms/config/confile.py +++ b/pyms/config/confile.py @@ -1,6 +1,7 @@ """Module to read yaml or json conf""" import logging import re +from typing import Dict, Union, Text, Tuple, Generator, Iterable import anyconfig @@ -16,7 +17,9 @@ class ConfFile(dict): """Recursive get configuration from dictionary, a config file in JSON or YAML format from a path or `CONFIGMAP_FILE` environment variable. **Atributes:** + * path: Path to find the `DEFAULT_CONFIGMAP_FILENAME` and `DEFAULT_KEY_FILENAME` if use encrypted vars * empty_init: Allow blank variables + * config: Allow to pass a dictionary to ConfFile without use a file """ _empty_init = False @@ -47,27 +50,40 @@ def __init__(self, *args, **kwargs): super(ConfFile, self).__init__(config) - def to_flask(self): + def to_flask(self) -> Dict: return ConfFile(config={k.upper(): v for k, v in self.items()}) - def set_config(self, config): + def set_config(self, config: Dict) -> Dict: + """ + Set a dictionary as attributes of ConfFile. This attributes could be access as `ConfFile["attr"]` or + ConfFile.attr + :param config: a dictionary from `config.yml` + :return: + """ config = dict(self.normalize_config(config)) + pop_encripted_keys = [] for k, v in config.items(): if k.lower().startswith("enc_"): k_not_crypt = re.compile(re.escape('enc_'), re.IGNORECASE) setattr(self, k_not_crypt.sub('', k), self._crypt.decrypt(v)) + pop_encripted_keys.append(k) else: setattr(self, k, v) + + # Delete encrypted keys to prevent decrypt multiple times a element + for x in pop_encripted_keys: + config.pop(x) + return config - def normalize_config(self, config): + def normalize_config(self, config: Dict) -> Iterable[Tuple[Text, Union[Dict, Text, bool]]]: for key, item in config.items(): if isinstance(item, dict): item = ConfFile(config=item, empty_init=self._empty_init) yield self.normalize_keys(key), item @staticmethod - def normalize_keys(key): + def normalize_keys(key: Text) -> Text: """The keys will be transformed to a attribute. We need to replace the charactes not valid""" key = key.replace("-", "_") return key @@ -90,6 +106,10 @@ def __getattr__(self, name, *args, **kwargs): raise AttrDoesNotExistException("Variable {} not exist in the config file".format(name)) def reload(self): + """ + Remove file from memoize variable, return again the content of the file and set the configuration again + :return: None + """ config_src = self._loader.reload(anyconfig.load) self.set_config(config_src) From 09e17c0e6d774ee14e1513f33d1fd041f877d82d Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 17 Feb 2020 18:53:42 +0100 Subject: [PATCH 06/23] Fix, send path to ConfFile from Microservice class to set path from code --- Pipfile.lock | 26 +++++++++++++------------- pyms/flask/app/create_app.py | 2 +- pyms/utils/files.py | 21 +++++++++++++-------- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index f0eb2c5..46e49b9 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "202ec3deff30d8c4b65ba37d8244ef5b350bd9d0508697e84ce4ed61107d1820" + "sha256": "b240135d1529558346e1da859a0fb45340c7bf7b6896e1e47f3e24f3d70ca1a0" }, "pipfile-spec": 6, "requires": { @@ -511,17 +511,17 @@ }, "gitdb2": { "hashes": [ - "sha256:0a84c85d07166cefaf10fa4b728a8a1b847a7f0d081e2fbe083185462158c126", - "sha256:51380b9d8b977b42ac9fc3a8e17af90a56ffc573ca9c6b79f99033b04f29f457" + "sha256:0375d983fd887d03c8942e81b1b0abc6c320cfb500cd3fe0d9c0eac87fbf2b52", + "sha256:b2b3a67090c17dc61f8407ca485e79ae811225ab5ebcd98ac5ee01448e8987b5" ], - "version": "==3.0.0" + "version": "==3.0.2" }, "gitpython": { "hashes": [ - "sha256:99c77677f31f255e130f3fed4c8e0eebb35f1a09df98ff965fff6774f71688cf", - "sha256:99cd0403cecd8a13b95d2e045b9fcaa7837137fcc5ec3105f2c413305d82c143" + "sha256:620b3c729bbc143b498cfea77e302999deedc55faec5b1067086c9ef90e101bc", + "sha256:a43a5d88a5bbc3cf32bb5223e4b4e68fd716db5e9996cad6e561bbfee6e5f4af" ], - "version": "==3.0.7" + "version": "==3.0.8" }, "googleapis-common-protos": { "hashes": [ @@ -668,11 +668,11 @@ }, "mkdocs-material": { "hashes": [ - "sha256:1d486635b03f5a2ec87325842f7b10c7ae7daa0eef76b185572eece6a6ea212c", - "sha256:7f3afa0a09c07d0b89a6a9755fdb00513aee8f0cec3538bb903325c80f66f444" + "sha256:73d3f5e23db9191e9a9d02278c71444c6ac35a8107d02001c31d2c184c0f105f", + "sha256:df0d460a67447c3f5753fe02e07ce67b42a5107957bebb1b19bb0bca9449ce84" ], "index": "pypi", - "version": "==4.6.3" + "version": "==5.0.0b1" }, "more-itertools": { "hashes": [ @@ -878,11 +878,11 @@ }, "tox": { "hashes": [ - "sha256:5c45d08f1dcc9bc97cdea3e5a69c8f4ad042cc37cbe6cf53e126f4a9005b7d3b", - "sha256:73e2ade68cd71a2765ee739ecc27c8c92e9b9c09acbad0f2c5307b20214e0d13" + "sha256:0cbe98369081fa16bd6f1163d3d0b2a62afa29d402ccfad2bd09fb2668be0956", + "sha256:676f1e3e7de245ad870f956436b84ea226210587d1f72c8dfb8cd5ac7b6f0e70" ], "index": "pypi", - "version": "==3.14.4" + "version": "==3.14.5" }, "typed-ast": { "hashes": [ diff --git a/pyms/flask/app/create_app.py b/pyms/flask/app/create_app.py index aa57c68..e12de78 100644 --- a/pyms/flask/app/create_app.py +++ b/pyms/flask/app/create_app.py @@ -99,7 +99,7 @@ def example(): def __init__(self, *args, **kwargs): self.path = os.path.dirname(kwargs.get("path", __file__)) validate_conf() - self.config = get_conf(service=CONFIG_BASE) + self.config = get_conf(path=self.path, service=CONFIG_BASE) self.init_services() def init_services(self) -> None: diff --git a/pyms/utils/files.py b/pyms/utils/files.py index c1530c8..caf1afa 100644 --- a/pyms/utils/files.py +++ b/pyms/utils/files.py @@ -38,19 +38,24 @@ def _get_conf_from_env(self, fn=None): return self._get_conf_from_file(fn) def _get_conf_from_file(self, fn=None): - if not self.path or not os.path.isfile(self.path): - logger.debug("File {} NOT FOUND".format(self.path)) + path = self.path + + if path and os.path.isdir(path): + path = os.path.join(path, self.default_file) + + if not path or not os.path.isfile(path): + logger.debug("File {} NOT FOUND".format(path)) return {} - if self.path not in files_cached: - logger.debug("[CONF] Configmap {} found".format(self.path)) + if path not in files_cached: + logger.debug("[CONF] Configmap {} found".format(path)) if fn: - files_cached[self.path] = fn(self.path) + files_cached[path] = fn(path) else: - file_to_read = open(self.path, 'rb') + file_to_read = open(path, 'rb') content = file_to_read.read() # The key will be type bytes file_to_read.close() - files_cached[self.path] = content - return files_cached[self.path] + files_cached[path] = content + return files_cached[path] def reload(self, fn=None): files_cached.pop(self.path, None) From 30fa3fb8e9726deaf3a7b1982adfef4cda2b722b Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 17 Feb 2020 19:06:33 +0100 Subject: [PATCH 07/23] Updated pipfile --- Pipfile.lock | 12 ++++++------ README.md | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 46e49b9..e64d190 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -380,10 +380,10 @@ }, "zipp": { "hashes": [ - "sha256:5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50", - "sha256:d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e" + "sha256:6f181bdb1a8c8019f8c11517680f7c1a836a7274c40de9165abfd6da228e54f2", + "sha256:fddb41c555ab338cdf27bc1d92cc6e3c05db8d1f1e7ba89d9646976702367333" ], - "version": "==2.2.0" + "version": "==2.2.1" } }, "develop": { @@ -940,10 +940,10 @@ }, "zipp": { "hashes": [ - "sha256:5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50", - "sha256:d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e" + "sha256:6f181bdb1a8c8019f8c11517680f7c1a836a7274c40de9165abfd6da228e54f2", + "sha256:fddb41c555ab338cdf27bc1d92cc6e3c05db8d1f1e7ba89d9646976702367333" ], - "version": "==2.2.0" + "version": "==2.2.1" } } } diff --git a/README.md b/README.md index fdf062c..087412f 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![Requirements Status](https://requires.io/github/python-microservices/pyms/requirements.svg?branch=master)](https://requires.io/github/python-microservices/pyms/requirements/?branch=master) [![Total alerts](https://img.shields.io/lgtm/alerts/g/python-microservices/pyms.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/python-microservices/pyms/alerts/) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/python-microservices/pyms.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/python-microservices/pyms/context:python) +[![Documentation Status](https://readthedocs.org/projects/py-ms/badge/?version=latest)](https://py-ms.readthedocs.io/en/latest/?badge=latest) [![Gitter](https://img.shields.io/gitter/room/DAVFoundation/DAV-Contributors.svg)](https://gitter.im/python-microservices/pyms) From 19745f77d54e2ac5cb42672a92bb9f4ea5a7e767 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 17 Feb 2020 19:32:24 +0100 Subject: [PATCH 08/23] Fix unused import --- pyms/config/confile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyms/config/confile.py b/pyms/config/confile.py index 7f1df27..fdf4b0d 100644 --- a/pyms/config/confile.py +++ b/pyms/config/confile.py @@ -1,7 +1,7 @@ """Module to read yaml or json conf""" import logging import re -from typing import Dict, Union, Text, Tuple, Generator, Iterable +from typing import Dict, Union, Text, Tuple, Iterable import anyconfig From 2a6cc487c80a38bd8627887b594e8165ee002738 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 17 Feb 2020 19:58:05 +0100 Subject: [PATCH 09/23] Updated dependencies --- Pipfile | 6 ++-- Pipfile.lock | 91 ++++++++++++++++++++++++++-------------------------- 2 files changed, 48 insertions(+), 49 deletions(-) diff --git a/Pipfile b/Pipfile index 2c47ba2..35de821 100644 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,7 @@ python-json-logger = ">=0.1.10" pyyaml = ">=5.1.2" anyconfig = ">=0.9.8" swagger-ui-bundle = ">=0.0.2" -connexion = {extras = ["swagger-ui"],version = "==2.4.0"} +connexion = {extras = ["swagger-ui"],version = "==2.6.0"} jaeger-client = "==4.3.0" flask-opentracing = "*" opentracing = ">=2.1" @@ -19,7 +19,7 @@ cryptography = "*" [dev-packages] requests-mock = "*" -coverage = "==4.5.4" +coverage = "==5.0.3" pytest = "*" pytest-cov = "*" pylint = "*" @@ -28,7 +28,7 @@ tox = "*" bandit = "*" mkdocs = "*" mkdocs-material = "*" -lightstep = "==4.3.0" +lightstep = "==4.4.3" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index e64d190..a51c10e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "b240135d1529558346e1da859a0fb45340c7bf7b6896e1e47f3e24f3d70ca1a0" + "sha256": "5634661dbc510a146c1825a5287a3a3b74ec94b0af9e5740c7ff2b24f57e7012" }, "pipfile-spec": 6, "requires": { @@ -96,11 +96,11 @@ "swagger-ui" ], "hashes": [ - "sha256:6e0569b646f2e6229923dc4e4c6e0325e223978bd19105779fd81e16bcb22fdf", - "sha256:7b4268e9ea837241e530738b35040345b78c8748d05d2c22805350aca0cd5b1c" + "sha256:bf32bfae6af337cfa4a8489c21516adbe5c50e3f8dc0b7ed2394ce8dde218018", + "sha256:c568e579f84be808e387dcb8570bb00a536891be1318718a0dad3ba90f034191" ], "index": "pypi", - "version": "==2.4.0" + "version": "==2.6.0" }, "contextlib2": { "hashes": [ @@ -380,10 +380,10 @@ }, "zipp": { "hashes": [ - "sha256:6f181bdb1a8c8019f8c11517680f7c1a836a7274c40de9165abfd6da228e54f2", - "sha256:fddb41c555ab338cdf27bc1d92cc6e3c05db8d1f1e7ba89d9646976702367333" + "sha256:12248a63bbdf7548f89cb4c7cda4681e537031eda29c02ea29674bc6854460c2", + "sha256:7c0f8e91abc0dc07a5068f315c52cb30c66bfbc581e5b50704c8a2f6ebae794a" ], - "version": "==2.2.1" + "version": "==3.0.0" } }, "develop": { @@ -445,41 +445,40 @@ }, "coverage": { "hashes": [ - "sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", - "sha256:0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650", - "sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5", - "sha256:19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d", - "sha256:23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351", - "sha256:245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755", - "sha256:331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef", - "sha256:386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca", - "sha256:3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca", - "sha256:60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9", - "sha256:63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc", - "sha256:6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5", - "sha256:6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f", - "sha256:7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe", - "sha256:826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888", - "sha256:93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5", - "sha256:9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce", - "sha256:af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5", - "sha256:bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e", - "sha256:bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e", - "sha256:c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9", - "sha256:dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437", - "sha256:df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1", - "sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c", - "sha256:e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24", - "sha256:e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47", - "sha256:eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2", - "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28", - "sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c", - "sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7", - "sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0", - "sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025" + "sha256:15cf13a6896048d6d947bf7d222f36e4809ab926894beb748fc9caa14605d9c3", + "sha256:1daa3eceed220f9fdb80d5ff950dd95112cd27f70d004c7918ca6dfc6c47054c", + "sha256:1e44a022500d944d42f94df76727ba3fc0a5c0b672c358b61067abb88caee7a0", + "sha256:25dbf1110d70bab68a74b4b9d74f30e99b177cde3388e07cc7272f2168bd1477", + "sha256:3230d1003eec018ad4a472d254991e34241e0bbd513e97a29727c7c2f637bd2a", + "sha256:3dbb72eaeea5763676a1a1efd9b427a048c97c39ed92e13336e726117d0b72bf", + "sha256:5012d3b8d5a500834783689a5d2292fe06ec75dc86ee1ccdad04b6f5bf231691", + "sha256:51bc7710b13a2ae0c726f69756cf7ffd4362f4ac36546e243136187cfcc8aa73", + "sha256:527b4f316e6bf7755082a783726da20671a0cc388b786a64417780b90565b987", + "sha256:722e4557c8039aad9592c6a4213db75da08c2cd9945320220634f637251c3894", + "sha256:76e2057e8ffba5472fd28a3a010431fd9e928885ff480cb278877c6e9943cc2e", + "sha256:77afca04240c40450c331fa796b3eab6f1e15c5ecf8bf2b8bee9706cd5452fef", + "sha256:7afad9835e7a651d3551eab18cbc0fdb888f0a6136169fbef0662d9cdc9987cf", + "sha256:9bea19ac2f08672636350f203db89382121c9c2ade85d945953ef3c8cf9d2a68", + "sha256:a8b8ac7876bc3598e43e2603f772d2353d9931709345ad6c1149009fd1bc81b8", + "sha256:b0840b45187699affd4c6588286d429cd79a99d509fe3de0f209594669bb0954", + "sha256:b26aaf69713e5674efbde4d728fb7124e429c9466aeaf5f4a7e9e699b12c9fe2", + "sha256:b63dd43f455ba878e5e9f80ba4f748c0a2156dde6e0e6e690310e24d6e8caf40", + "sha256:be18f4ae5a9e46edae3f329de2191747966a34a3d93046dbdf897319923923bc", + "sha256:c312e57847db2526bc92b9bfa78266bfbaabac3fdcd751df4d062cd4c23e46dc", + "sha256:c60097190fe9dc2b329a0eb03393e2e0829156a589bd732e70794c0dd804258e", + "sha256:c62a2143e1313944bf4a5ab34fd3b4be15367a02e9478b0ce800cb510e3bbb9d", + "sha256:cc1109f54a14d940b8512ee9f1c3975c181bbb200306c6d8b87d93376538782f", + "sha256:cd60f507c125ac0ad83f05803063bed27e50fa903b9c2cfee3f8a6867ca600fc", + "sha256:d513cc3db248e566e07a0da99c230aca3556d9b09ed02f420664e2da97eac301", + "sha256:d649dc0bcace6fcdb446ae02b98798a856593b19b637c1b9af8edadf2b150bea", + "sha256:d7008a6796095a79544f4da1ee49418901961c97ca9e9d44904205ff7d6aa8cb", + "sha256:da93027835164b8223e8e5af2cf902a4c80ed93cb0909417234f4a9df3bcd9af", + "sha256:e69215621707119c6baf99bda014a45b999d37602cb7043d943c76a59b05bf52", + "sha256:ea9525e0fef2de9208250d6c5aeeee0138921057cd67fcef90fbed49c4d62d37", + "sha256:fca1669d464f0c9831fd10be2eef6b86f5ebd76c724d1e0706ebdff86bb4adf0" ], "index": "pypi", - "version": "==4.5.4" + "version": "==5.0.3" }, "distlib": { "hashes": [ @@ -593,11 +592,11 @@ }, "lightstep": { "hashes": [ - "sha256:15912e3bbd7e9b00f106a5a83362f5b3994e17008abb04a3e16b49ff81eca310", - "sha256:5e3c98e4f2a5fa7205c6f1d87dfd13ae9dd7990ab152b6d8fc5992e3847a3c3e" + "sha256:4196a378ed10b8af7b5609f9225211da62c763bc28d2d34b111bdff9548c14ec", + "sha256:88b00309496dddcf546fa54f7dd2b5c04a686f14e0c50bca8a40686caff087f8" ], "index": "pypi", - "version": "==4.3.0" + "version": "==4.4.3" }, "livereload": { "hashes": [ @@ -940,10 +939,10 @@ }, "zipp": { "hashes": [ - "sha256:6f181bdb1a8c8019f8c11517680f7c1a836a7274c40de9165abfd6da228e54f2", - "sha256:fddb41c555ab338cdf27bc1d92cc6e3c05db8d1f1e7ba89d9646976702367333" + "sha256:12248a63bbdf7548f89cb4c7cda4681e537031eda29c02ea29674bc6854460c2", + "sha256:7c0f8e91abc0dc07a5068f315c52cb30c66bfbc581e5b50704c8a2f6ebae794a" ], - "version": "==2.2.1" + "version": "==3.0.0" } } } From 0a0e81535aa325c73f11239464bfab77e4206584 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 17 Feb 2020 21:35:38 +0100 Subject: [PATCH 10/23] Changed SHA2576 to SHA512_256 --- pyms/utils/crypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyms/utils/crypt.py b/pyms/utils/crypt.py index e18968b..94c0a04 100644 --- a/pyms/utils/crypt.py +++ b/pyms/utils/crypt.py @@ -20,7 +20,7 @@ def generate_key(self, password: Text, write_to_file: bool = False): password = password.encode() # Convert to type bytes salt = os.urandom(16) kdf = PBKDF2HMAC( - algorithm=hashes.SHA256(), + algorithm=hashes.SHA512_256(), length=32, salt=salt, iterations=100000, From 15cdc7351eb498c63ff9cb6f4245380c21f84699 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 17 Feb 2020 22:18:19 +0100 Subject: [PATCH 11/23] Updated docs --- docs/configuration.md | 4 +- docs/encrypt_decryt_configuration.md | 98 ++++++++++++++++++++++++++++ docs/index.md | 1 + 3 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 docs/encrypt_decryt_configuration.md diff --git a/docs/configuration.md b/docs/configuration.md index 5b68d74..6d04ac8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -4,8 +4,8 @@ **CONFIGMAP_FILE**: The path to the configuration file. By default, PyMS search the configuration file in your actual folder with the name "config.yml" -**CONFIGMAP_SERVICE**: the name of the keyword that define the block of key-value of [Flask Configuration Handling](http://flask.pocoo.org/docs/1.0/config/) -and your own configuration (see the next section to more info) +**KEY_FILE**: The path to the key file to decrypt your configuration. By default, PyMS search the configuration file in your +actual folder with the name "key.key" ## Create configuration Each microservice needs a config file in yaml or json format to work with it. This configuration contains diff --git a/docs/encrypt_decryt_configuration.md b/docs/encrypt_decryt_configuration.md new file mode 100644 index 0000000..0dc68c9 --- /dev/null +++ b/docs/encrypt_decryt_configuration.md @@ -0,0 +1,98 @@ +# Encrypt Configuration + +When you work in multiple environments: local, dev, testing, production... you must set critical configuration in your +variables, like: + +config.yml, for local propose: +```yaml +pyms: + config: + DEBUG: true + TESTING: true + APPLICATION_ROOT : "" + SECRET_KEY: "gjr39dkjn344_!67#" + SQLALCHEMY_DATABASE_URI: mysql+mysqlconnector://user_of_db:user_of_db@localhost/my_schema +``` + +config_pro.yml, for production environment: +```yaml +pyms: + config: + DEBUG: true + TESTING: true + APPLICATION_ROOT : "" + SECRET_KEY: "gjr39dkjn344_!67#" + SQLALCHEMY_DATABASE_URI: mysql+mysqlconnector://important_user:****@localhost/my_schema +``` + +You can move this file to a [Kubernetes secret](https://kubernetes.io/docs/concepts/configuration/secret/), +use [Vault](https://learn.hashicorp.com/vault) or encrypt the configuration with [AWS KMS](https://aws.amazon.com/en/kms/) + or [Google KMS](https://cloud.google.com/kms). We strongly recommended this ways to encrypt/decrypt your configuration, + but i you want a no vendor locking option or you haven`t the resources to use this methods, we create a way to encrypt + and decrypt your variables. + +## 1. Generate a key +PyMS have a command line option to create a key file. this key is created with [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard). +You can run the terminal: + +```bash +pyms create-key +``` + +Type a password and this command create a file called `key.key`. This file contains a unique key. If you lost this file +and re-run the create command, the key hash will be different and your code encrypted with this key can`t be decrypted. + +Store the key in a secure site, and NOT COMMIT this key to your repository. + + +## 2. Add your key to your environment + +Move, for example, your key to `mv key.key /home/my_user/keys/myproject.key` + +then, store this key in a environment variable with: + +```bash +export KEY_FILE=/home/my_user/keys/myproject.key +``` + +## 3. Encrypt your information and put in config + +Do you remember the example file `config_pro.yml`? now you can encrypt and decrypt the information, you can run the command +`pyms encrypt [string]` to generate a crypt string, for example: + +```bash +pyms encrypt 'mysql+mysqlconnector://important_user:****@localhost/my_schema' +>> Encrypted OK: b'gAAAAABeSwBJv43hnGAWZOY50QjBX6uGLxUb3Q6fcUhMxKspIVIco8qwwZvxRg930uRlsd47isroXzkdRRnb4-x2dsQMp0dln8Pm2ySHH7TryLbQYEFbSh8RQK7zor-hX6gB-JY3uQD3IMtiVKx9AF95D6U4ydT-OA==' +``` + +And put this string in your `config_pro.yml`: +```yaml +pyms: + config: + DEBUG: true + TESTING: true + APPLICATION_ROOT : "" + SECRET_KEY: "gjr39dkjn344_!67#" + ENC_SQLALCHEMY_DATABASE_URI: gAAAAABeSwBJv43hnGAWZOY50QjBX6uGLxUb3Q6fcUhMxKspIVIco8qwwZvxRg930uRlsd47isroXzkdRRnb4-x2dsQMp0dln8Pm2ySHH7TryLbQYEFbSh8RQK7zor-hX6gB-JY3uQD3IMtiVKx9AF95D6U4ydT-OA== +``` + +Do you see the difference between `ENC_SQLALCHEMY_DATABASE_URI` and `SQLALCHEMY_DATABASE_URI`? In the next step you +can find the answer + +## 4. Decrypt from your config file + +Pyms knows if a variable is encrypted if this var start with the prefix `enc_` or `ENC_`. PyMS search for your key file +in the `KEY_FILE` env variable and decrypt this value and store it in the same variable without the `enc_` prefix, +por example, + +```yaml +ENC_SQLALCHEMY_DATABASE_URI: gAAAAABeSwBJv43hnGAWZOY50QjBX6uGLxUb3Q6fcUhMxKspIVIco8qwwZvxRg930uRlsd47isroXzkdRRnb4-x2dsQMp0dln8Pm2ySHH7TryLbQYEFbSh8RQK7zor-hX6gB-JY3uQD3IMtiVKx9AF95D6U4ydT-OA== +``` + +Will be stored as + +```bash +SQLALCHEMY_DATABASE_URI: mysql+mysqlconnector://user_of_db:user_of_db@localhost/my_schema +``` + +And you can access to this var with `current_app.config["SQLALCHEMY_DATABASE_URI"]` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 184b1e3..5871108 100644 --- a/docs/index.md +++ b/docs/index.md @@ -40,6 +40,7 @@ Nowadays, is not perfect and we have a looong roadmap, but we hope this library * [Installation](installation.md) * [Quickstart](quickstart.md) * [Configuration](configuration.md) +* [Encrypt/Decrypt Configuration](encrypt_decryt_configuration.md) * [Services](services.md) * [PyMS structure](structure.md) * [Microservice class](ms_class.md) From 3107bf8e8afe5fce0a4224e3da1f5e10a1df58f5 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 17 Feb 2020 22:21:57 +0100 Subject: [PATCH 12/23] Fix example --- examples/microservice_configuration/main.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/microservice_configuration/main.py b/examples/microservice_configuration/main.py index 3b8a679..f82613c 100644 --- a/examples/microservice_configuration/main.py +++ b/examples/microservice_configuration/main.py @@ -2,8 +2,4 @@ app = ms.create_app() if __name__ == '__main__': - """ - run first: - export CONFIGMAP_SERVICE=my-configure-microservice - """ app.run() From 3d38c7699790a9407435f96e666096f339f096d3 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Tue, 18 Feb 2020 08:34:09 +0100 Subject: [PATCH 13/23] Fix docs and update examples --- docs/configuration.md | 1 + docs/services.md | 64 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6d04ac8..5d67e80 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,6 +11,7 @@ actual folder with the name "key.key" Each microservice needs a config file in yaml or json format to work with it. This configuration contains the Flask settings of your project and the [Services](services.md). With this way to create configuration files, we solve two problems of the [12 Factor apps](https://12factor.net/): + - Store config out of the code - Dev/prod parity: the configuration could be injected and not depends of our code, for example, Kubernetes configmaps diff --git a/docs/services.md b/docs/services.md index c50782b..e859ac6 100644 --- a/docs/services.md +++ b/docs/services.md @@ -5,46 +5,106 @@ This services are created as an attribute of the [Microservice class](ms_class.m To add a service check the [configuration section](configuration.md). +You can declare a service but activate/deactivate with the keyword `enabled`, like: + +```yaml +pyms: + services: + requests: + enabled: false +``` + Current services are: ## Swagger / connexion + Extends the Microservice with [Connexion](https://github.com/zalando/connexion) and [swagger-ui](https://github.com/sveint/flask-swagger-ui). + ### Configuration + The parameters you can add to your config are: + * **path:** The relative or absolute route to your swagger yaml file. The default value is the current directory * **file:** The name of you swagger yaml file. The default value is `swagger.yaml` * **url:** The url where swagger run in your server. The default value is `/ui/`. * **project_dir:** Relative path of the project folder to automatic routing, + see [this link for more info](https://github.com/zalando/connexion#automatic-routing). The default value is `project`. +### Example + +```yaml +pyms: + services: + swagger: + path: "swagger" + file: "swagger.yaml" + url: "/ui/" + project_dir: "project.views" +``` + ## Requests + Extend the [requests library](http://docs.python-requests.org/en/master/) with trace headers and parsing JSON objects. Encapsulate common rest operations between business services propagating trace headers if set up. + ### Configuration + The parameters you can add to your config are: + * **data:** wrap the response in a data field of an envelope object, and add other meta data to that wrapper. The default value is None * **retries:** If the response is not correct, send again the request. The default number of retries is 3. * **status_retries:** List of response status code that consider "not correct". The default values are [500, 502, 504] * **propagate_headers:** Propagate the headers of the actual execution to the request. The default values is False +### Example + +```yaml +pyms: + services: + requests: + data: "data" + retries: 4 + status_retries: [400, 401, 402, 403, 404, 405, 500, 501, 502, 503] + propagate_headers: true +``` + ## Tracer -Add trace to all executions with[opentracing](https://github.com/opentracing-contrib/python-flask). + +Add trace to all executions with [opentracing](https://github.com/opentracing-contrib/python-flask). + ### Configuration + The parameters you can add to your config are: + * **client:** set the client to use traces, The actual options are [Jaeger](https://github.com/jaegertracing/jaeger-client-python) and [Lightstep](https://github.com/lightstep/lightstep-tracer-python). The default value is jaeger. +* **host:** The url to send the data of traces. Check [this tutorial](https://opentracing.io/guides/python/quickstart/) to create your own server +* **component_name:** The name of your application to show in Prometheus metrics + +### Example + +```yaml +pyms: + services: + tracer: + client: "jaeger" + host: "localhost" + component_name: "Python Microservice" +``` ## Metrics Adds [Prometheus](https://prometheus.io/) metrics using the [Prometheus Client Library](https://github.com/prometheus/client_python). At the moment, the next metrics are available: + - Incoming requests latency as a histogram - Incoming requests number as a counter, divided by HTTP method, endpoint and HTTP status - Total number of log events divided by level - If `tracer` service activated and it's jaeger, it will show its metrics -To use this service, you may add the next to you configuration file: +### Example ```yaml pyms: From eee12436cfc628baed7bbd874e01b778c8661f1c Mon Sep 17 00:00:00 2001 From: avara1986 Date: Tue, 18 Feb 2020 13:42:11 +0100 Subject: [PATCH 14/23] Fix issue #88 --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 2c1adf4..817c18e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,7 +12,7 @@ nav: - Structure: structure.md - Microservice class: ms_class.md - Examples: examples.md - - Routing: installation.md + - Routing: routing.md - Structure of a microservice project: structure_project.md theme: name: 'material' \ No newline at end of file From 3cea656df9fbf2a477812fd143a9352b9ecbce57 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 19 Feb 2020 09:09:56 +0100 Subject: [PATCH 15/23] Update docs/encrypt_decryt_configuration.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Àlex Pérez --- docs/encrypt_decryt_configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/encrypt_decryt_configuration.md b/docs/encrypt_decryt_configuration.md index 0dc68c9..572c382 100644 --- a/docs/encrypt_decryt_configuration.md +++ b/docs/encrypt_decryt_configuration.md @@ -28,7 +28,7 @@ pyms: You can move this file to a [Kubernetes secret](https://kubernetes.io/docs/concepts/configuration/secret/), use [Vault](https://learn.hashicorp.com/vault) or encrypt the configuration with [AWS KMS](https://aws.amazon.com/en/kms/) or [Google KMS](https://cloud.google.com/kms). We strongly recommended this ways to encrypt/decrypt your configuration, - but i you want a no vendor locking option or you haven`t the resources to use this methods, we create a way to encrypt + but if you want a no vendor locking option or you haven`t the resources to use this methods, we create a way to encrypt and decrypt your variables. ## 1. Generate a key @@ -95,4 +95,4 @@ Will be stored as SQLALCHEMY_DATABASE_URI: mysql+mysqlconnector://user_of_db:user_of_db@localhost/my_schema ``` -And you can access to this var with `current_app.config["SQLALCHEMY_DATABASE_URI"]` \ No newline at end of file +And you can access to this var with `current_app.config["SQLALCHEMY_DATABASE_URI"]` From f25730e33254557df29423e4d68844d33e66155c Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 19 Feb 2020 09:10:04 +0100 Subject: [PATCH 16/23] Update docs/encrypt_decryt_configuration.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Àlex Pérez --- docs/encrypt_decryt_configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/encrypt_decryt_configuration.md b/docs/encrypt_decryt_configuration.md index 572c382..7a4120f 100644 --- a/docs/encrypt_decryt_configuration.md +++ b/docs/encrypt_decryt_configuration.md @@ -33,7 +33,7 @@ use [Vault](https://learn.hashicorp.com/vault) or encrypt the configuration with ## 1. Generate a key PyMS have a command line option to create a key file. this key is created with [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard). -You can run the terminal: +You can run the next command in the terminal: ```bash pyms create-key From 405640be31438498b3e78ca5c2dff67228eaaf0d Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 19 Feb 2020 09:10:11 +0100 Subject: [PATCH 17/23] Update docs/encrypt_decryt_configuration.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Àlex Pérez --- docs/encrypt_decryt_configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/encrypt_decryt_configuration.md b/docs/encrypt_decryt_configuration.md index 7a4120f..cc82bc7 100644 --- a/docs/encrypt_decryt_configuration.md +++ b/docs/encrypt_decryt_configuration.md @@ -39,7 +39,7 @@ You can run the next command in the terminal: pyms create-key ``` -Type a password and this command create a file called `key.key`. This file contains a unique key. If you lost this file +Then, type a password and it will create a file called `key.key`. This file contains a unique key. If you loose this file and re-run the create command, the key hash will be different and your code encrypted with this key can`t be decrypted. Store the key in a secure site, and NOT COMMIT this key to your repository. From 22cad3ee32d0c1781835549e2914f2fad0de0b3c Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 19 Feb 2020 09:10:19 +0100 Subject: [PATCH 18/23] Update docs/encrypt_decryt_configuration.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Àlex Pérez --- docs/encrypt_decryt_configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/encrypt_decryt_configuration.md b/docs/encrypt_decryt_configuration.md index cc82bc7..4a55de5 100644 --- a/docs/encrypt_decryt_configuration.md +++ b/docs/encrypt_decryt_configuration.md @@ -57,7 +57,7 @@ export KEY_FILE=/home/my_user/keys/myproject.key ## 3. Encrypt your information and put in config -Do you remember the example file `config_pro.yml`? now you can encrypt and decrypt the information, you can run the command +Do you remember the example file `config_pro.yml`? Now you can encrypt and decrypt the information, you can run the command `pyms encrypt [string]` to generate a crypt string, for example: ```bash From d2190419a03ab95ff7af9d92e9303a5339c06ec5 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 19 Feb 2020 09:10:27 +0100 Subject: [PATCH 19/23] Update docs/encrypt_decryt_configuration.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Àlex Pérez --- docs/encrypt_decryt_configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/encrypt_decryt_configuration.md b/docs/encrypt_decryt_configuration.md index 4a55de5..0bfa083 100644 --- a/docs/encrypt_decryt_configuration.md +++ b/docs/encrypt_decryt_configuration.md @@ -81,7 +81,7 @@ can find the answer ## 4. Decrypt from your config file -Pyms knows if a variable is encrypted if this var start with the prefix `enc_` or `ENC_`. PyMS search for your key file +Pyms knows if a variable is encrypted if this var start with the prefix `enc_` or `ENC_`. PyMS searchs for your key file in the `KEY_FILE` env variable and decrypt this value and store it in the same variable without the `enc_` prefix, por example, From 98da3a06cc51dd1f38003935a8601fe51b8c8716 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 19 Feb 2020 09:10:39 +0100 Subject: [PATCH 20/23] Update docs/encrypt_decryt_configuration.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Àlex Pérez --- docs/encrypt_decryt_configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/encrypt_decryt_configuration.md b/docs/encrypt_decryt_configuration.md index 0bfa083..a081cca 100644 --- a/docs/encrypt_decryt_configuration.md +++ b/docs/encrypt_decryt_configuration.md @@ -40,7 +40,7 @@ pyms create-key ``` Then, type a password and it will create a file called `key.key`. This file contains a unique key. If you loose this file -and re-run the create command, the key hash will be different and your code encrypted with this key can`t be decrypted. +and re-run the create command, the key hash will be different and your code encrypted with this key won't be able to be decrypted. Store the key in a secure site, and NOT COMMIT this key to your repository. From 8b08e2e3b6443208bc922c5a133c9050e0168da4 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 19 Feb 2020 09:20:26 +0100 Subject: [PATCH 21/23] Update docs/encrypt_decryt_configuration.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Àlex Pérez --- docs/encrypt_decryt_configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/encrypt_decryt_configuration.md b/docs/encrypt_decryt_configuration.md index a081cca..7ee655d 100644 --- a/docs/encrypt_decryt_configuration.md +++ b/docs/encrypt_decryt_configuration.md @@ -32,7 +32,7 @@ use [Vault](https://learn.hashicorp.com/vault) or encrypt the configuration with and decrypt your variables. ## 1. Generate a key -PyMS have a command line option to create a key file. this key is created with [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard). +PyMS have a command line option to create a key file. This key is created with [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard). You can run the next command in the terminal: ```bash From cc7fc7815a29f088031591363b347a39596a70a4 Mon Sep 17 00:00:00 2001 From: albertovara Date: Wed, 19 Feb 2020 09:44:58 +0100 Subject: [PATCH 22/23] Fix typo --- docs/encrypt_decryt_configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/encrypt_decryt_configuration.md b/docs/encrypt_decryt_configuration.md index 7ee655d..a865f6b 100644 --- a/docs/encrypt_decryt_configuration.md +++ b/docs/encrypt_decryt_configuration.md @@ -32,7 +32,7 @@ use [Vault](https://learn.hashicorp.com/vault) or encrypt the configuration with and decrypt your variables. ## 1. Generate a key -PyMS have a command line option to create a key file. This key is created with [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard). +PyMS has a command line option to create a key file. this key is created with [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard). You can run the next command in the terminal: ```bash From f2522a42927ae7aaaca0efd882002635df1e7761 Mon Sep 17 00:00:00 2001 From: albertovara Date: Wed, 19 Feb 2020 09:49:44 +0100 Subject: [PATCH 23/23] Fix typo --- docs/encrypt_decryt_configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/encrypt_decryt_configuration.md b/docs/encrypt_decryt_configuration.md index a865f6b..b8347af 100644 --- a/docs/encrypt_decryt_configuration.md +++ b/docs/encrypt_decryt_configuration.md @@ -32,7 +32,7 @@ use [Vault](https://learn.hashicorp.com/vault) or encrypt the configuration with and decrypt your variables. ## 1. Generate a key -PyMS has a command line option to create a key file. this key is created with [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard). +PyMS has a command line option to create a key file. This key is created with [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard). You can run the next command in the terminal: ```bash