diff --git a/.github/workflows/scs-compliance-check.yml b/.github/workflows/scs-compliance-check.yml index 1a1a6f0b0..7b6a7b099 100644 --- a/.github/workflows/scs-compliance-check.yml +++ b/.github/workflows/scs-compliance-check.yml @@ -33,9 +33,7 @@ jobs: password: ${{ secrets[inputs.secret_name] }} EOF - name: "Run scs-compliance-check" - run: "cd /scs-compliance && ./scs-compliance-check.py scs-compatible-${{ inputs.layer }}.yaml --version ${{ inputs.version }} -o result.yaml" - env: - OS_CLOUD: ${{ inputs.cloud }} + run: "cd /scs-compliance && ./scs-compliance-check.py scs-compatible-${{ inputs.layer }}.yaml --version ${{ inputs.version }} -o result.yaml -s ${{ inputs.cloud }} -a os_cloud=${{ inputs.cloud }}" - name: "Upload results" uses: actions/upload-artifact@v3 with: diff --git a/Standards/scs-0003-v1-sovereign-cloud-standards-yaml.md b/Standards/scs-0003-v1-sovereign-cloud-standards-yaml.md index 2244b9ed3..23a1eb348 100644 --- a/Standards/scs-0003-v1-sovereign-cloud-standards-yaml.md +++ b/Standards/scs-0003-v1-sovereign-cloud-standards-yaml.md @@ -89,7 +89,14 @@ The certification YAML _MUST_ contain the following keys: The certification YAML _MAY_ contain the following keys: -### Prerequisite +| Key | Type | Description | +| ------------------- | ------------------ | ------------------------------------------------------------- | +| `prerequisite` | Map | Descriptor for the prerequisite certificate scope (see below) | +| `variables` | Array of String | Lists variables that may occur in check tool descriptors | + +The main check tool will expect an assignment for these variables (which is specific to the subject under test), and every occurrence of the variable in the check tool descriptor will be substituted accordingly. + +### Prerequisite descriptor A certificate within a certain level (above SCS-compatible) can only be granted if a valid corresponding certificate of the level below is presented, where corresponding means: of the same layer. The latter certificate is said to be a prerequisite for the former. @@ -97,11 +104,10 @@ where corresponding means: of the same layer. The latter certificate is said to We implement this logic by allowing for the designation of a certificate scope as a prerequisite; then a certificate of that prerequisite scope has to be presented before the certificate of the scope in question can be granted. -| Key | Type | Description | Example | -| ------------------- | ------ | ------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| `prerequisite` | Map | Descriptor for the prerequisite certificate scope, if any | | -| `prerequisite.name` | String | Full name of the certificate scope | _SCS Compatible IaaS_ | -| `prerequisite.url` | String | Valid URL to the latest raw version of the certificate scope | _[scs-compatible-iaas.yaml](https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Tests/scs-compatible-iaas.yaml)_ | +| Key | Type | Description | Example | +| ------ | ------ | ------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| `name` | String | Full name of the certificate scope | _SCS Compatible IaaS_ | +| `url` | String | Valid URL to the latest raw version of the certificate scope | _[scs-compatible-iaas.yaml](https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Tests/scs-compatible-iaas.yaml)_ | ### Version descriptor @@ -138,10 +144,15 @@ Every list of standards consists of several standards that – altogether – de | Key | Type | Description | Example | | ----------------- | ------ | ---------------------------------------------------------------------------------------------------------------------------------------- | ---------------------- | | `executable` | String | Valid local filename (relative to the path of scs-compliance-check.py) of a script that verifies compliance with the particular standard | _image-md-check.py_ | -| `args` | String | _Optional_ list of arguments to be passed to the `check_tool`. Preferably none needed. | `-v` | +| `env` | Map | _Optional_ key-value map of environment variables (values may use variables) | OS_CLOUD: {os_cloud} | +| `args` | String | _Optional_ command-line arguments to be passed to the `check_tool` (may use variables) | `-v -k {kubeconfig}` | | `condition` | String | _Optionally_ overrides the per-standard condition (`mandatory` or `optional`) | _optional_ | | `classification` | String | One of: `light` (_default_), `medium`, `heavy`; describes the resource usage of the script; used to select an appropiate test interval | _heavy_ | +As mentioned, variables may be used within `env` and `args`; they are enclosed in single braces, like so: `{var}`. +If a brace is desired, it needs to be doubled: `{{` will be turned into `{`. When the main check tool is run, +each occurrence of a variable will be substituted for according to the variable assignment for the subject under test. + _Note_: the `executable` could in principle also be given via a URL; however, this is not yet supported due to security considerations. ### Basic Example @@ -152,6 +163,8 @@ url: https://raw.githubusercontent.com/SovereignCloudStack/Docs/main/Certificati prerequisite: name: SCS Compatible IaaS url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Tests/scs-compatible-iaas.yaml +variables: + - os_cloud versions: - version: v5 # This version is in a draft state and work in progress # No stabilized_at: date set yet @@ -161,12 +174,14 @@ versions: condition: mandatory # is default and can be left out check_tools: - executable: flavor-name-check.py + env: + OS_CLOUD: "{os_cloud}" - name: Image metadata url: https://raw.githubusercontent.com/SovereignCloudStack/Docs/main/Standards/SCS-0004-v1-image-metadata.md condition: mandatory check_tools: - executable: image-md-check.py - args: -v + args: -c {os_cloud} -v - executable: image-md-check2.py condition: optional - version: v4 # This is the upcoming version with a given target date. No further changes should be done to this set of standards @@ -185,8 +200,6 @@ versions: obsoleted_at: 2021-11-01 standards: - name: .... -kaas: - - ... ``` ## Process diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index 291fddd05..0f540d0d0 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -1,5 +1,7 @@ name: SCS Compatible IaaS url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Tests/scs-compatible-iaas.yaml +variables: + - os_cloud versions: - version: v4 stabilized_at: 2024-02-28 @@ -11,28 +13,28 @@ versions: url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0100-v3-flavor-naming.md check_tools: - executable: ./iaas/flavor-naming/flavor-names-openstack.py - args: "--mand=./iaas/scs-0100-v3-flavors.yaml" + args: -c {os_cloud} --mand=./iaas/scs-0100-v3-flavors.yaml # Note: --v2plus would outlaw the v1 flavor names. Don't do this yet. - name: Entropy url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0101-v1-entropy.md check_tools: - executable: ./iaas/entropy/entropy-check.py - args: -d + args: -c {os_cloud} -d - name: Image metadata url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0102-v1-image-metadata.md check_tools: - executable: ./iaas/image-metadata/image-md-check.py - args: -s -v + args: -c {os_cloud} -s -v - name: Standard flavors url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0103-v1-standard-flavors.md check_tools: - executable: ./iaas/standard-flavors/flavors-openstack.py - args: -d "./iaas/scs-0103-v1-flavors.yaml" + args: -c {os_cloud} -d ./iaas/scs-0103-v1-flavors.yaml - name: Standard images url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0104-v1-standard-images.md check_tools: - executable: ./iaas/standard-images/images-openstack.py - args: -d "./iaas/scs-0104-v1-images.yaml" + args: -c {os_cloud} -d ./iaas/scs-0104-v1-images.yaml - version: v3 stabilized_at: 2023-06-15 obsoleted_at: 2024-04-30 @@ -41,13 +43,13 @@ versions: url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0100-v3-flavor-naming.md check_tools: - executable: ./iaas/flavor-naming/flavor-names-openstack.py - args: "--v3" + args: --v3 -c {os_cloud} # Note: "--v3 --v2plus" would outlaw the v1 flavor names. Don't do this yet. - name: Image metadata url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0102-v1-image-metadata.md check_tools: - executable: ./iaas/image-metadata/image-md-check.py - args: -v + args: -v -c {os_cloud} - name: OpenStack Powered Compute v2022.11 url: https://opendev.org/openinfra/interop/src/branch/master/guidelines/2022.11.json - version: v2 @@ -58,11 +60,12 @@ versions: url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0100-v2-flavor-naming.md check_tools: - executable: ./iaas/flavor-naming/flavor-names-openstack.py + args: -c {os_cloud} - name: Image metadata url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0102-v1-image-metadata.md check_tools: - executable: ./iaas/image-metadata/image-md-check.py - args: -v + args: -v -c {os_cloud} - name: OpenStack Powered Compute v2022.11 url: https://opendev.org/openinfra/interop/src/branch/master/guidelines/2022.11.json - version: v1 @@ -73,11 +76,11 @@ versions: url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0100-v1-flavor-naming.md check_tools: - executable: ./iaas/flavor-naming/flavor-names-openstack.py - args: "--v1prefer" + args: -c {os_cloud} --v1prefer - name: Image metadata url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0102-v1-image-metadata.md check_tools: - executable: ./iaas/image-metadata/image-md-check.py - args: -v + args: -c {os_cloud} -v - name: OpenStack Powered Compute v2020.11 url: https://opendev.org/openinfra/interop/src/branch/master/guidelines/2020.11.json diff --git a/Tests/scs-compliance-check.py b/Tests/scs-compliance-check.py index c80675467..cc5473c26 100755 --- a/Tests/scs-compliance-check.py +++ b/Tests/scs-compliance-check.py @@ -5,18 +5,13 @@ # # (c) Eduard Itrich # (c) Kurt Garloff +# (c) Matthias Büchse # SPDX-License-Identifier: Apache-2.0 -"""Master SCS compliance checker +"""Main SCS compliance checker reads SCS certification requirements from e.g. scs-compatible.yaml and performs all the checks for the specified level and outputs a -verdict from all tests (which is reflected in the exit code) -The tests require the OpenStack SDK (and in the future probably -also k8s python bindings) to be installed and access to IaaS -(for the iaas layer tests) via configure clouds/secure.yaml -which are passed in OS_CLOUD (or --os-cloud cmdline param). -In the future als access to a cluster with KUBECONFIG pointing -to a working file. +verdict from all tests (which is reflected in the exit code). The goal is to work without any special admin privileges. (If we find things that can't be tested as normal user, we would split these tests out.) @@ -38,10 +33,10 @@ # valid keywords for various parts of the spec, to be checked using `check_keywords` KEYWORDS = { - 'spec': ('name', 'url', 'versions', 'prerequisite'), + 'spec': ('name', 'url', 'versions', 'prerequisite', 'variables'), 'version': ('version', 'standards', 'stabilized_at', 'obsoleted_at'), 'standard': ('check_tools', 'url', 'name', 'condition'), - 'checktool': ('executable', 'args', 'condition', 'classification'), + 'checktool': ('executable', 'env', 'args', 'condition', 'classification'), } @@ -50,12 +45,12 @@ def usage(file=sys.stdout): print("""Usage: scs-compliance-check.py [options] compliance-spec.yaml Options: -v/--verbose: More verbose output -q/--quiet: Don't output anything but errors - -s/--single-scope: Don't perform required checks for prerequisite scopes -d/--date YYYY-MM-DD: Check standards valid on specified date instead of today -V/--version VERS: Force version VERS of the standard (instead of deriving from date) - -c/--os-cloud CLOUD: Use specified cloud env (instead of OS_CLOUD env var) + -s/--subject SUBJECT: Name of the subject (cloud) under test, for the report -o/--output REPORT_PATH: Generate yaml report of compliance check under given path -C/--critical-only: Only return critical errors in return code + -a/--assign KEY=VALUE: assign variable to be used for the run (as required by yaml file) With -C, the return code will be nonzero precisely when the tests couldn't be run to completion. """.strip(), file=file) @@ -89,7 +84,8 @@ def __init__(self): self.arg0 = None self.verbose = False self.quiet = False - self.os_cloud = os.environ.get("OS_CLOUD") + self.subject = "" + self.assignment = {} self.checkdate = datetime.date.today() self.version = None self.output = None @@ -99,9 +95,9 @@ def __init__(self): def apply_argv(self, argv): """Parse options. May exit the program.""" try: - opts, args = getopt.gnu_getopt(argv, "hvqd:V:sc:o:r:C", ( + opts, args = getopt.gnu_getopt(argv, "hvqd:V:s:o:r:Ca:", ( "help", "verbose", "quiet", "date=", "version=", - "os-cloud=", "output=", "resource-usage=", "critical-only" + "subject=", "output=", "resource-usage=", "critical-only", "assign", )) except getopt.GetoptError as exc: print(f"Option error: {exc}", file=sys.stderr) @@ -119,14 +115,19 @@ def apply_argv(self, argv): self.checkdate = datetime.date.fromisoformat(opt[1]) elif opt[0] == "-V" or opt[0] == "--version": self.version = opt[1] - elif opt[0] == "-c" or opt[0] == "--os-cloud": - self.os_cloud = opt[1] + elif opt[0] == "-s" or opt[0] == "--subject": + self.subject = opt[1] elif opt[0] == "-o" or opt[0] == "--output": self.output = opt[1] elif opt[0] == "-r" or opt[0] == "--resource-usage": self.classes = [x.strip() for x in opt[1].split(",")] elif opt[0] == "-C" or opt[0] == "--critical-only": self.critical_only = True + elif opt[0] == "-a" or opt[0] == "--assign": + key, value = opt[1].split("=", 1) + if key in self.assignment: + raise ValueError(f"Double assignment for {key!r}") + self.assignment[key] = value else: print(f"Error: Unknown argument {opt[0]}", file=sys.stderr) if len(args) < 1: @@ -163,10 +164,10 @@ def suppress(*args, **kwargs): return -def invoke_check_tool(check, check_env, check_cwd): +def invoke_check_tool(exe, args, env, cwd): """run check tool and return invokation dict to use in the report""" try: - compl = run_check_tool(check["executable"], check.get("args", ''), env=check_env, cwd=check_cwd) + compl = run_check_tool(exe, args, env, cwd) except Exception as e: invokation = { "rc": 127, @@ -191,15 +192,22 @@ def invoke_check_tool(check, check_env, check_cwd): def main(argv): """Entry point for the checker""" config = Config() - config.apply_argv(argv) - if not config.os_cloud: - print("You need to have OS_CLOUD set or pass --os-cloud=CLOUD.", file=sys.stderr) + try: + config.apply_argv(argv) + except Exception as exc: + print(f"CRITICAL: {exc}", file=sys.stderr) + return 1 + if not config.subject: + print("You need pass --subject=SUBJECT.", file=sys.stderr) return 1 printv = suppress if not config.verbose else partial(print, file=sys.stderr) printnq = suppress if config.quiet else partial(print, file=sys.stderr) with open(config.arg0, "r", encoding="UTF-8") as specfile: spec = yaml.load(specfile, Loader=yaml.SafeLoader) - check_env = {'OS_CLOUD': config.os_cloud, **os.environ} + missing_vars = [v for v in spec.get("variables", ()) if v not in config.assignment] + if missing_vars: + print(f"Missing variable assignments (via -a) for: {', '.join(missing_vars)}") + return 1 check_cwd = os.path.dirname(config.arg0) or os.getcwd() allaborts = 0 allerrors = 0 @@ -207,15 +215,15 @@ def main(argv): "spec": copy.deepcopy(spec), "run": { "argv": argv, - "os_cloud": config.os_cloud, - # TODO: Add kubeconfig context as well + "subject": config.subject, + "assignment": config.assignment, "checked_at": config.checkdate, "classes": config.classes, "forced_version": config.version or None, "aborts": 0, "errors": 0, "versions": {}, - "invokations": {}, + "invocations": {}, }, } check_keywords('spec', spec) @@ -224,7 +232,7 @@ def main(argv): if "prerequisite" in spec: print("WARNING: prerequisite not yet implemented!", file=sys.stderr) vrs = report["run"]["versions"] - memo = report["run"]["invokations"] # memoize check tool results + memo = report["run"]["invocations"] # memoize check tool results matches = 0 for vd in spec["versions"]: check_keywords('version', vd) @@ -237,7 +245,7 @@ def main(argv): "passed": False, "aborts": 0, "errors": 0, - "invokations": [], + "invocations": [], } if outdated and not config.version: continue @@ -252,7 +260,7 @@ def main(argv): file=sys.stderr) errors = 0 aborts = 0 - invokations = vr["invokations"] + invocations = vr["invocations"] for standard in vd.get("standards", ()): check_keywords('standard', standard) optional = condition_optional(standard) @@ -266,15 +274,18 @@ def main(argv): if check.get("classification", "light") not in config.classes: print(f"skipping check tool '{check['executable']}' because of resource classification") continue - args = check.get('args', '') - memo_key = f"{check['executable']} {args}".strip() + args = check.get('args', '').format(**config.assignment) + env = {key: value.format(**config.assignment) for key, value in check.get('env', {}).items()} + env_str = " ".join(f"{key}={value}" for key, value in env.items()) + memo_key = f"{env_str} {check['executable']} {args}".strip() invokation = memo.get(memo_key) if invokation is None: - invokation = invoke_check_tool(check, check_env, check_cwd) + check_env = {**os.environ, **env} + invokation = invoke_check_tool(check["executable"], args, check_env, check_cwd) printv("\n".join(invokation["stdout"])) printnq("\n".join(invokation["stderr"])) memo[memo_key] = invokation - invokations.append(memo_key) + invocations.append(memo_key) abort = invokation["critical"] error = invokation["error"] printnq(f"... returned {error} errors, {abort} aborts") @@ -285,7 +296,7 @@ def main(argv): vr["errors"] = errors vr["passed"] = not (aborts + errors) printnq("*******************************************************") - printnq(f"Verdict for os_cloud {config.os_cloud}, {spec['name']}, " + printnq(f"Verdict for subject {config.subject}, {spec['name']}, " f"version {vd['version']}: {errcode_to_text(aborts + errors)}") allaborts += aborts allerrors += errors diff --git a/playbooks/compliance_check.yaml b/playbooks/compliance_check.yaml index 2d436383f..203028b55 100644 --- a/playbooks/compliance_check.yaml +++ b/playbooks/compliance_check.yaml @@ -4,7 +4,7 @@ tasks: - name: Run compliance script ansible.builtin.shell: | - python3 ~/Tests/scs-compliance-check.py ~/Tests/scs-compatible-iaas.yaml -c {{ cloud }} -o {{ cloud }}-iaas.yaml -C + python3 ~/Tests/scs-compliance-check.py ~/Tests/scs-compatible-iaas.yaml -s {{ cloud }} -a os_cloud={{ cloud }} -o {{ cloud }}-iaas.yaml -C register: result changed_when: true # failed_when: result.rc != 0