Addded support for config_merger filtering of root properties

README.md

@@ -343,6 +343,51 @@ Leveraging HIML, the config-merger script loads the configs tree structure and d
 
 Under each level, there is a mandatory "level key" that is used by config-merger for computing the end result. This key should be present in one of the files under each level. (eg. env.yaml under env).
 
+### Output filtering schema
+
+Some configs that are specified in the higher levels of the directory tree might not be needed in the end (leaf) result. For this reason, the config-merger script supports a filtering schema that can be specified via the `--filter-schema-key` parameter. This property must be present in the config and contains rules for removing root level keys from the output. The rules are applied for specified levels (explicit value or regex) of the config tree and will keep the keys that match the list or a regex pattern.
+
+
+```yaml
+# intermediate config after hierarchical merge
+env: dev
+cluster: cluster1
+region: us-east-1
+key1: persisted
+key2: dropped
+keep_1: persisted
+tags:
+  cost_center: 123
+_schema:
+- level: env
+  value: dev
+  keys:
+    values:
+    - key1
+    regex: "keep_.*"
+- level: cluster
+  value: cluster1
+  keys:
+    values:
+    - tags
+```
+
+Build the output with filtering:
+```sh
+himl-config-merger examples/filters --output-dir merged_output --levels env region cluster --leaf-directories cluster --filter-schema-key _schema
+```
+
+```yaml
+# output after filtering
+env: dev
+cluster: cluster1
+region: us-east-1
+key1: persisted
+keep_1: persisted
+tags:
+  cost_center: 123
+```
+
 ### Extra merger features
 
 Apart from the standard features found in the `PyYaml` library, the `himl-config-merger` component also implements a custom YAML tag called `!include`.

examples/filters/default.yaml

@@ -0,0 +1,67 @@
+env: default
+region: default
+cluster: default
+
+cluster_info:
+  name: default # this will be overridden by the inner cluster.yaml file
+
+  # Interpolation example
+  description: "This is cluster: {{cluster}}. It is using {{cluster_info.node_type}} instance type."
+  node_type: c3.2xlarge # default value, which can be overridden by each cluster
+cluster_metrics:
+  - id: 1
+    metric: cpu
+    value: 90
+  - id: 2
+    metric: memory
+    value: 90
+  - id: 3
+    metric: disk
+    value: 90
+metrics:
+  - cpu
+  - memory
+  - disk
+myList:
+ - id1
+ - id4
+# Fetching the secret value at runtime, from a secrets store (in this case AWS SSM).
+# passphrase: "{{ssm.path(/key/coming/from/aws/secrets/store/manager).aws_profile(myprofile)}}"
+
+# Fetching the value at runtime from S3
+# my_secret: "{{s3.bucket(my-bucket).path(path/to/file.txt).base64encode(true).aws_profile(myprofile)}}"
+
+
+_schema:
+  # Keep _schema key for all outputs
+  # - level: region
+  #   keys:
+  #     values:
+  #     - "_schema"
+
+  - level: cluster
+    regex: "cluster.*"
+    keys:
+      values:
+      - persisted_key
+      # - persisted_key_referenced
+      # - persisted_key_to_drop
+      # - persisted_key_to_drop2
+      # - cluster_persisted_object
+      # - cluster_persisted_list
+
+  - level: cluster
+    value: cluster1
+    keys:
+      values:
+        - testkey
+        - home
+        - cluster_persisted_key
+
+  - level: cluster
+    value: cluster2
+    keys:
+      values:
+        - metrics
+        - myList
+      regex: ".*persisted.*"

examples/filters/env=dev/env.yaml

@@ -0,0 +1,6 @@
+env: dev
+persisted_key: &persisted persisted key
+dropped_key: &dropped object will be filtered out
+persisted_key_referenced: *persisted
+persisted_key_to_drop: *dropped
+persisted_key_to_drop2: *dropped

examples/filters/env=dev/region=us-east-1/cluster=cluster1/cluster.yaml

@@ -0,0 +1,21 @@
+cluster: cluster1
+
+testkey: |-
+  # Set to true to log user information returned from LDAP
+    verbose_logging = true
+
+    [[servers]]
+    # Ldap server host
+    host = "someaddress"
+
+    # Default port is 389 or 636 if use_ssl = true
+    port = 389
+
+    start_tls = true
+
+cluster_persisted_key: this object will be persisted
+cluster_filtered_key: this object will be filtered out
+cluster_persisted_list: "{{ myList }}"
+cluster_persisted_object:
+  cluster_info: "{{ cluster_info }}"
+  cluster_list: "{{ myList }}"

examples/filters/env=dev/region=us-east-1/cluster=cluster2/cluster.yaml

@@ -0,0 +1,23 @@
+cluster: cluster2
+cluster_metrics:
+  - id: 1
+    metric: cpu
+    value: 95
+  - id: 2
+    metric: memory
+    value: 95
+  - id: 3
+    metric: disk
+    remove: True
+  - metric: exec
+    value: 5
+metrics:
+  - cpu
+  - exec
+myList:
+ - id1
+ - id2
+ - id3
+persisted_key: this object will be persisted
+dropped_key: this object will be dropped
+another_persisted_key: this object will also be persisted

examples/filters/env=dev/region=us-east-1/region.yaml

@@ -0,0 +1 @@
+region: us-east-1

examples/filters/env=dev/region=us-west-2/cluster=cluster1/cluster.yaml

@@ -0,0 +1,2 @@
+cluster: cluster1
+home: "{{env(HOME)}}"

examples/filters/env=dev/region=us-west-2/region.yaml

@@ -0,0 +1 @@
+region: us-west-2

examples/filters/env=prod/env.yaml

@@ -0,0 +1 @@
+env: prod

examples/filters/env=prod/region=eu-west-2/cluster=ireland1/cluster.yaml

@@ -0,0 +1,3 @@
+cluster: ireland1
+
+file: "{{cwd}}/test.txt"

examples/filters/env=prod/region=eu-west-2/region.yaml

@@ -0,0 +1 @@
+region: eu-west-2

himl/config_merger.py

@@ -16,7 +16,7 @@
 import yaml
 from .config_generator import ConfigProcessor
 from multiprocessing import Pool, cpu_count
-
+from .schema_filters import Schema
 logger = logging.getLogger(__name__)
 
 
@@ -71,7 +71,7 @@ def __traverse_path(self, path: str, yaml_dict: dict):
                 current_key, yaml_dict))
 
 
-def merge_configs(directories, levels, output_dir, enable_parallel):
+def merge_configs(directories, levels, output_dir, enable_parallel, filter_schema):
     """
     Method for running the merge configuration logic under different formats
     :param directories: list of paths for leaf directories
@@ -82,7 +82,7 @@ def merge_configs(directories, levels, output_dir, enable_parallel):
     config_processor = ConfigProcessor()
     process_config = []
     for path in directories:
-        process_config.append((config_processor, path, levels, output_dir))
+        process_config.append((config_processor, path, levels, output_dir, filter_schema))
 
     if enable_parallel:
         logger.info("Processing config in parallel")
@@ -102,6 +102,7 @@ def merge_logic(process_params):
     path = process_params[1]
     levels = process_params[2]
     output_dir = process_params[3]
+    filter_schema = process_params[4]
 
     # load the !include tag
     Loader.add_constructor('!include', Loader.include)
@@ -121,6 +122,12 @@ def merge_logic(process_params):
     if not os.path.exists(publish_path):
         os.makedirs(publish_path)
 
+    if filter_schema:
+        if filter_schema not in output:
+            raise Exception("Filter schema key not found in config")
+        schema = Schema(output[filter_schema], levels)
+        schema.filter(output)
+
     # create the yaml file for output using the publish_path and last level_values element
     filename = "{0}/{1}.yaml".format(publish_path, level_values[-1])
     logger.info("Found input config directory: %s", path)
@@ -171,6 +178,8 @@ def parser_options(args):
                         help='leaf directories, for instance: cluster', required=True)
     parser.add_argument('--enable-parallel', dest='enable_parallel', default=False,
                         action='store_true', help='Process config using multiprocessing')
+    parser.add_argument('--filter-schema-key', dest='filter_schema', default=None, type=str,
+                        help='keep these keys from the generated data, based on the configured schema key')
     return parser.parse_args(args)
 
 
@@ -182,4 +191,4 @@ def run(args=None):
 
     # merge the configs using HIML
     merge_configs(dirs, opts.hierarchy_levels,
-                  opts.output_dir, opts.enable_parallel)
+                  opts.output_dir, opts.enable_parallel, opts.filter_schema)

himl/main.py

@@ -37,8 +37,8 @@ def do_run(self, opts):
             opts.print_data = True
 
         config_processor = ConfigProcessor()
-                                 
-        config_processor.process(cwd, opts.path, filters, excluded_keys, opts.enclosing_key, opts.remove_enclosing_key,
+
+        config_processor.process(cwd, opts.path, filters, filter_config, excluded_keys, opts.enclosing_key, opts.remove_enclosing_key,
                                  opts.output_format, opts.print_data, opts.output_file, opts.skip_interpolation_resolving,
                                  opts.skip_interpolation_validation, opts.skip_secrets, opts.multi_line_string,
                                  type_strategies= [(list, [opts.merge_list_strategy.value]), (dict, ["merge"])] )

himl/schema_filters.py

@@ -0,0 +1,39 @@
+import re
+
+
+class Schema(object):
+
+    def __init__(self, filter_schema, levels):
+        self.filter_schema = filter_schema
+        self.levels = levels
+
+    def filter(self, output):
+
+        removable_keys = set(output.keys()) - set(self.levels)
+
+        for filter in self.filter_schema:
+            level = filter.get("level")
+            if not level:
+                raise Exception("Filter schema must contain a level : {}".format(filter))
+            if level not in self.levels:
+                raise Exception("Filter schema level must be one of the levels : {}".format(self.levels))
+
+            if "value" in filter:
+                predicate = lambda v: v == filter["value"]
+            elif "regex" in filter:
+                level_re = re.compile(filter["regex"])
+                predicate = lambda v: level_re.match(v)
+            else:
+                predicate = lambda v: True
+            if not predicate(output[level]):
+                continue
+
+            keys = filter.get("keys")
+            if "values" in keys:
+                removable_keys = removable_keys - set(keys["values"])
+            if "regex" in keys:
+                key_re = re.compile(keys["regex"])
+                removable_keys = {k for k in removable_keys if not key_re.match(k)}
+
+        for key in removable_keys:
+            del output[key]