From 88241ac8e8b18bc51cc716f2a5064c4e4068d8b3 Mon Sep 17 00:00:00 2001
From: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
Date: Fri, 23 Feb 2024 13:38:10 +0100
Subject: [PATCH 01/13] Add requirements for k8s-version-recency-check.py

Previously, they have been installed as transitive dependencies of the other
listed packages and thus, have only been available by coincidence.

Signed-off-by: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
---
 Tests/requirements.in  |  3 +++
 Tests/requirements.txt | 12 +++++++++---
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/Tests/requirements.in b/Tests/requirements.in
index 07ccfe674..128fed5c0 100644
--- a/Tests/requirements.in
+++ b/Tests/requirements.in
@@ -1,5 +1,8 @@
+aiohttp
 click
 fabric
 kubernetes_asyncio
+python-dateutil
 PyYAML
 openstacksdk
+requests
diff --git a/Tests/requirements.txt b/Tests/requirements.txt
index 21c9e55c9..bbbb9ebf1 100644
--- a/Tests/requirements.txt
+++ b/Tests/requirements.txt
@@ -5,7 +5,9 @@
 #    pip-compile requirements.in
 #
 aiohttp==3.9.3
-    # via kubernetes-asyncio
+    # via
+    #   -r requirements.in
+    #   kubernetes-asyncio
 aiosignal==1.3.1
     # via aiohttp
 appdirs==1.4.4
@@ -94,14 +96,18 @@ pycparser==2.21
 pynacl==1.5.0
     # via paramiko
 python-dateutil==2.8.2
-    # via kubernetes-asyncio
+    # via
+    #   -r requirements.in
+    #   kubernetes-asyncio
 pyyaml==6.0.1
     # via
     #   -r requirements.in
     #   kubernetes-asyncio
     #   openstacksdk
 requests==2.31.0
-    # via keystoneauth1
+    # via
+    #   -r requirements.in
+    #   keystoneauth1
 requestsexceptions==1.4.0
     # via openstacksdk
 six==1.16.0

From a325ea91826eabf553db6e9db59e2547d9df0be1 Mon Sep 17 00:00:00 2001
From: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
Date: Wed, 28 Feb 2024 07:36:17 +0100
Subject: [PATCH 02/13] Refactor and prepare code for additions of v2

* Move to an importable Python module for tests
* Add pytest and pytest-asyncio test requirements
* Add K8sVersion and K8sRelease data classes instead of K8sVersionInfo
* Convert CVEVersionInfo into a dataclass
* Use the VersionRange concept
* Compare based on release age with timedelta
* Add support for kubeconfig contexts
* Add EOL data for K8s versions
* Update script name to reflect new standard name
* Escape the dots in regular expressions
* Retrieve the CVE id directly

Signed-off-by: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
---
 .../config.yaml.template                      |   8 +-
 .../kaas/k8s-version-policy/k8s-eol-data.yml  |  12 +
 .../k8s_version_policy.py}                    | 284 ++++++++----------
 Tests/test-requirements.in                    |   3 +
 Tests/test-requirements.txt                   |  22 ++
 5 files changed, 162 insertions(+), 167 deletions(-)
 rename Tests/kaas/{k8s-version-recency => k8s-version-policy}/config.yaml.template (78%)
 create mode 100644 Tests/kaas/k8s-version-policy/k8s-eol-data.yml
 rename Tests/kaas/{k8s-version-recency/k8s-version-recency-check.py => k8s-version-policy/k8s_version_policy.py} (55%)
 create mode 100644 Tests/test-requirements.in
 create mode 100644 Tests/test-requirements.txt

diff --git a/Tests/kaas/k8s-version-recency/config.yaml.template b/Tests/kaas/k8s-version-policy/config.yaml.template
similarity index 78%
rename from Tests/kaas/k8s-version-recency/config.yaml.template
rename to Tests/kaas/k8s-version-policy/config.yaml.template
index 1392dad87..e7a247c8d 100644
--- a/Tests/kaas/k8s-version-recency/config.yaml.template
+++ b/Tests/kaas/k8s-version-policy/config.yaml.template
@@ -5,20 +5,20 @@ logging:
   version: 1
   disable_existing_loggers: False
   formatters:
-    k8s-version-recency-check:
+    k8s_version_policy:
       format: "%(levelname)s: %(message)s"
   handlers:
     console:
       class: logging.StreamHandler
-      formatter: k8s-version-recency-check
+      formatter: k8s_version_policy
       stream: ext://sys.stdout
     file:
       class: logging.handlers.WatchedFileHandler
-      formatter: k8s-version-recency-check
+      formatter: k8s_version_policy
       filename: MY-LOG-FILE-NAME.log
   root: # Configuring the default (root) logger is highly recommended
     handlers: [console]
   loggers:
-    k8s-version-recency-check:
+    k8s_version_policy:
       handlers: [console, file]
       propagate: no
\ No newline at end of file
diff --git a/Tests/kaas/k8s-version-policy/k8s-eol-data.yml b/Tests/kaas/k8s-version-policy/k8s-eol-data.yml
new file mode 100644
index 000000000..500e53ae0
--- /dev/null
+++ b/Tests/kaas/k8s-version-policy/k8s-eol-data.yml
@@ -0,0 +1,12 @@
+# https://kubernetes.io/releases/patch-releases/#detailed-release-history-for-active-branches
+# Please keep this list sorted descending by version.
+- version: '1.29'
+  end-of-life: '2025-02-28'
+- version: '1.28'
+  end-of-life: '2024-10-28'
+- version: '1.27'
+  end-of-life: '2024-06-28'
+- version: '1.26'
+  end-of-life: '2024-02-28'
+- version: '1.25'
+  end-of-life: '2023-10-28'
diff --git a/Tests/kaas/k8s-version-recency/k8s-version-recency-check.py b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
similarity index 55%
rename from Tests/kaas/k8s-version-recency/k8s-version-recency-check.py
rename to Tests/kaas/k8s-version-policy/k8s_version_policy.py
index 68e561b82..02ed8f8c0 100755
--- a/Tests/kaas/k8s-version-recency/k8s-version-recency-check.py
+++ b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
@@ -21,10 +21,11 @@
 License: CC-BY-SA 4.0
 """
 
+from dataclasses import dataclass
+from datetime import datetime, timedelta
 import aiohttp
 import asyncio
-import datetime
-from dateutil import relativedelta
+import contextlib
 import getopt
 import kubernetes_asyncio
 import logging
@@ -35,10 +36,9 @@
 import yaml
 
 
-MAJOR_VERSION_CADENCE = None
-MINOR_VERSION_CADENCE_MONTHS = 4
-PATCH_VERSION_CADENCE_WEEKS = 1
-CVE_VERSION_CADENCE_DAYS = 3
+MINOR_VERSION_CADENCE = timedelta(days=120)
+PATCH_VERSION_CADENCE = timedelta(weeks=1)
+CVE_VERSION_CADENCE = timedelta(days=3)
 CVE_SEVERITY = 8  # CRITICAL
 
 logging_config = {
@@ -46,14 +46,14 @@
     "version": 1,
     "disable_existing_loggers": False,
     "formatters": {
-        "k8s-version-recency-check": {
+        "k8s_version_policy": {
             "format": "%(levelname)s: %(message)s"
         }
     },
     "handlers": {
         "console": {
             "class": "logging.StreamHandler",
-            "formatter": "k8s-version-recency-check",
+            "formatter": "k8s_version_policy",
             "stream": "ext://sys.stdout"
         }
     },
@@ -83,7 +83,7 @@ def print_usage():
     print("""
 K8s Version Recency Compliance Check
 
-Usage: k8s-version-recency-check.py [-h] [-c|--config PATH/TO/CONFIG] -k|--kubeconfig PATH/TO/KUBECONFIG
+Usage: k8s_version_policy.py [-h] [-c|--config PATH/TO/CONFIG] -k|--kubeconfig PATH/TO/KUBECONFIG
 
 The K8s version recency check returns 0 if the version of the tested cluster is still acceptable, otherwise
 it returns 2 for an out-of date version or 3 if the used version should be updated due to a highly critical CVE.
@@ -147,95 +147,64 @@ def initialize_config(config):
     return config
 
 
-class K8sVersionInfo:
-    """Class that contains a k8s version info.
+@dataclass(order=True)
+class K8sVersion:
+    major: int
+    minor: int
+    patch: int = 0
 
-    Attributes:
-        major (int): Major version of the k8s version
-        minor (int): Minor version of the k8s version
-        patch (int): Patch version of the k8s version
-        date (datetime): release date of the k8s version
-    """
-    def __init__(self, major=0, minor=0, patch=0):
-        self.major = major
-        self.minor = minor
-        self.patch = patch
-
-        self.date = None
-
-    def __eq__(self, other):
-        if not isinstance(other, K8sVersionInfo):
-            raise TypeError
-        return self.major == other.major and self.minor == other.minor and self.patch == other.patch
-
-    def __gt__(self, other):
-        if not isinstance(other, K8sVersionInfo):
-            raise TypeError
-        patchcomp = self.minor == other.minor and self.patch > other.patch
-        return self.major > other.major or (self.major == other.major and (self.minor > other.minor or patchcomp))
-
-    def __ge__(self, other):
-        if not isinstance(other, K8sVersionInfo):
-            raise TypeError
-        patchcomp = self.minor == other.minor and self.patch >= other.patch
-        return self.major > other.major or (self.major == other.major and (self.minor > other.minor or patchcomp))
-
-    def __lt__(self, other):
-        if not isinstance(other, K8sVersionInfo):
-            raise TypeError
-        patchcomp = self.minor == other.minor and self.patch < other.patch
-        return self.major < other.major or (self.major == other.major and (self.minor < other.minor or patchcomp))
-
-    def __le__(self, other):
-        if not isinstance(other, K8sVersionInfo):
-            raise TypeError
-        patchcomp = self.minor == other.minor and self.patch <= other.patch
-        return self.major < other.major or (self.major == other.major and (self.minor < other.minor or patchcomp))
-
-    @classmethod
-    def extract_version(cls, string, separator=".", strip=None):
-        if strip is None:
-            strip = ["v"]
-        for s in strip:
-            string = string.strip(s)
-        components = string.strip().split(separator)
-        return cls(int(components[0]), int(components[1]), int(components[2]))
-
-    def check_for_version(self, major=None, minor=None, patch=None):
-        """Check if a version or part of the version is equal to the given version numbers"""
-        return (major is None or self.major == major) and \
-               (minor is None or self.minor == minor) and \
-               (patch is None or self.patch == patch)
+    def is_same_minor_version(self, other):
+        return self.major == other.major and self.minor == other.minor
 
     def __str__(self):
         return f"{self.major}.{self.minor}.{self.patch}"
 
 
-class CVEVersionInfo:
-    """Class that contains a CVE version info.
+def parse_version(version_str: str) -> K8sVersion:
+    cleansed = version_str.removeprefix("v").strip()
+    try:
+        major, minor, patch = cleansed.split(".")
+        return K8sVersion(int(major), int(minor), int(patch))
+    except ValueError:
+        raise ValueError(f"Unrecognized version format: {version_str}")
 
-    Attributes:
-        upper_version (K8sVersionInfo): Last version with the CVE
-        lower_version (K8sVersionInfo): First version with the CVE; this value will be set if either an affected version
-            is directly set in a CVE dataset or if the CVE dataset is in a non-standard format.
-            If the variable is set, `lower_version` and `upper_version` create a range of affected versions.
-        equal (bool): check if the version is equal to the `upper_version`, (less than is always checked, since the
-            format is build like this)
-    """
-    def __init__(self, lower_version, upper_version, equal=False):
-        self.lower_version = lower_version
-        self.upper_version = upper_version
 
-        self.equal = equal
+@dataclass
+class K8sRelease:
+    version: K8sVersion
+    released_at: datetime
+
+    def __str__(self):
+        return f"{self.version} ({self.released_at.isoformat()})"
+
+    @property
+    def age(self):
+        return datetime.now() - self.released_at
+
+
+def parse_github_release_data(release_data: dict) -> K8sRelease:
+    version = parse_version(release_data["tag_name"].split("-")[0])
+    released_at = datetime.strptime(release_data["published_at"], "%Y-%m-%dT%H:%M:%SZ")
+    return K8sRelease(version, released_at)
+
+
+@dataclass
+class VersionRange:
+    """Version range with an lower and upper bound."""
+
+    # First version with the CVE; this value will be set if either an affected
+    # version is directly set in a CVE dataset or if the CVE dataset is in a
+    # non-standard format.  If the variable is set, `lower_version` and
+    # `upper_version` create a range of affected versions.
+    lower_version: K8sVersion
+
+    # Last version with the CVE
+    upper_version: K8sVersion
 
-    def __eq__(self, other):
-        if not isinstance(other, CVEVersionInfo):
-            raise TypeError
-        return self.lower_version == other.lower_version and \
-            self.upper_version == other.upper_version and \
-            self.equal == self.equal
+    # True if upper_version is included in the range of affected versions
+    inclusive: bool = False
 
-    def is_version_affected(self, version_info):
+    def __contains__(self, version: K8sVersion) -> bool:
         # See the following link for more information about the format
         # https://www.cve.org/AllResources/CveServices#cve-json-5
 
@@ -243,31 +212,22 @@ def is_version_affected(self, version_info):
         if self.upper_version:
             # Check if a `lower version` exists and compare the version against it
             if self.lower_version:
-                gt = self.lower_version <= version_info
+                gt = self.lower_version <= version
             else:
                 gt = True
             # Compare the version either with `less than` or `less than or equal` against the `upper version`
-            if self.equal:
-                return gt and self.upper_version >= version_info
-            return gt and self.upper_version > version_info
+            if self.inclusive:
+                return gt and self.upper_version >= version
+            return gt and self.upper_version > version
         else:
             # If no upper version exists, we only need to check if the version is equal to the `lower version`
-            return self.lower_version == version_info
+            return self.lower_version == version
 
 
-def diff_months(date1, date2):
-    r = relativedelta.relativedelta(date2, date1)
-    return r.months + (12 * r.years)
-
-
-def diff_weeks(date1, date2):
-    delta = date1 - date2
-    return abs(delta.days / 7)
-
-
-def diff_days(date1, date2):
-    delta = date1 - date2
-    return abs(delta.days)
+@dataclass
+class ClusterInfo:
+    version: K8sVersion
+    name: str
 
 
 async def request_cve_data(session: aiohttp.ClientSession, cveid: str) -> dict:
@@ -279,39 +239,45 @@ async def request_cve_data(session: aiohttp.ClientSession, cveid: str) -> dict:
         return await resp.json()
 
 
-def parse_cve_version_information(cve_version_info):
+def parse_cve_version_information(cve_version_info: dict) -> VersionRange:
     """Parse the CVE version information according to their CVE JSON 5.0 schema"""
     vi_lower_version = None
     vi_upper_version = None
-    equal = False
+    inclusive = False
 
     # Extract the version if it is viable, but it's not a requirement
-    try:
-        vi_lower_version = K8sVersionInfo.extract_version(cve_version_info['version'])
-    except ValueError:
-        pass
+    with contextlib.suppress(ValueError):
+        vi_lower_version = parse_version(cve_version_info['version'])
 
     if 'lessThanOrEqual' in cve_version_info:
-        vi_upper_version = K8sVersionInfo.extract_version(cve_version_info['lessThanOrEqual'])
-        equal = True
+        vi_upper_version = parse_version(cve_version_info['lessThanOrEqual'])
+        inclusive = True
     elif 'lessThan' in cve_version_info:
-        vi_upper_version = K8sVersionInfo.extract_version(cve_version_info['lessThan'])
+        vi_upper_version = parse_version(cve_version_info['lessThan'])
 
     # This shouldn't happen, but if it happens, we look for non-standard descriptions
     # According to this(https://www.cve.org/AllResources/CveServices#cve-json-5),
     # this isn't how the data should be described
     if vi_lower_version is None and vi_upper_version is None:
-        if re.search(r'v?\d+.\d+.x', cve_version_info['version']):
+        if re.search(r'v?\d+\.\d+\.x', cve_version_info['version']):
             vdata = cve_version_info['version'].strip("v").split(".")
-            vi_lower_version = K8sVersionInfo(vdata[0], vdata[1], 0)
-            vi_upper_version = K8sVersionInfo(vdata[0], vdata[1], 0)
+            vi_lower_version = K8sVersion(int(vdata[0]), int(vdata[1]), 0)
+            vi_upper_version = K8sVersion(int(vdata[0]), int(vdata[1]), 0)
 
-        if re.search(r'v?\d+.\d+.\d+\s+-\s+v?\d+.\d+.\d+', cve_version_info['version']):
+        if re.search(r'v?\d+\.\d+\.\d+\s+-\s+v?\d+\.\d+\.\d+', cve_version_info['version']):
             vdata = cve_version_info['version'].split("-")
-            vi_lower_version = K8sVersionInfo.extract_version(vdata[0])
-            vi_upper_version = K8sVersionInfo.extract_version(vdata[1])
+            vi_lower_version = parse_version(vdata[0])
+            vi_upper_version = parse_version(vdata[1])
+
+    return VersionRange(vi_lower_version, vi_upper_version, inclusive)
 
-    return CVEVersionInfo(vi_lower_version, vi_upper_version, equal)
+
+def is_high_severity(cve_metrics: list) -> bool:
+    return any(
+        re.search(r'[cC][vV][sS]{1,2}V\d', metric_key) and metric_value['baseScore'] >= CVE_SEVERITY
+        for cve_metric in cve_metrics
+        for metric_key, metric_value in cve_metric.items()
+    )
 
 
 async def collect_cve_versions(session: aiohttp.ClientSession):
@@ -329,7 +295,7 @@ async def collect_cve_versions(session: aiohttp.ClientSession):
     ) as resp:
         cve_list = await resp.json()
 
-    tasks = [request_cve_data(session=session, cveid=cve['external_url'].split("=")[-1])
+    tasks = [request_cve_data(session=session, cveid=cve['id'])
              for cve in cve_list['items']]
 
     cve_data_list = await asyncio.gather(*tasks, return_exceptions=True)
@@ -344,6 +310,7 @@ async def collect_cve_versions(session: aiohttp.ClientSession):
             # https://github.com/CVEProject/cve-schema/tree/master/schema/v5.0
             # The containers -> cna path contains vulnerability information like severity, which is documented
             # under the metrics list.
+            # https://cveproject.github.io/cve-schema/schema/v5.0/docs/
         except KeyError as e:
             logger.debug(
                 f"They key {e} couldn't be found in the CVE json data for CVE "
@@ -351,13 +318,7 @@ async def collect_cve_versions(session: aiohttp.ClientSession):
             )
             continue
 
-        is_high_severity = any(
-            re.search(r'[cC][vV][sS]{1,2}V\d', metric_key) and metric_value['baseScore'] >= CVE_SEVERITY
-            for cve_metric in cve_metrics
-            for metric_key, metric_value in cve_metric.items()
-        )
-
-        if is_high_severity:
+        if is_high_severity(cve_metrics):
             affected_kubernetes_versions = [
                 parse_cve_version_information(version_info)
                 for aff in cve_affected
@@ -375,21 +336,18 @@ async def collect_cve_versions(session: aiohttp.ClientSession):
     return cfvs
 
 
-async def get_k8s_cluster_version(kubeconfig):
+async def get_k8s_cluster_info(kubeconfig, context=None) -> ClusterInfo:
     """Get the k8s version of the cluster under test."""
-    cluster_config = await kubernetes_asyncio.config.load_kube_config(kubeconfig)
+    cluster_config = await kubernetes_asyncio.config.load_kube_config(kubeconfig, context)
 
     async with kubernetes_asyncio.client.ApiClient() as api:
         version_api = kubernetes_asyncio.client.VersionApi(api)
-        ret = await version_api.get_code()
-
-        version = K8sVersionInfo.extract_version(ret.git_version)
-        version.date = datetime.datetime.strptime(ret.build_date, '%Y-%m-%dT%H:%M:%SZ')
+        response = await version_api.get_code()
+        version = parse_version(response.git_version)
+        return ClusterInfo(version, cluster_config.current_context['name'])
 
-        return version, cluster_config.current_context['name']
 
-
-def check_k8s_version_recency(version, cve_version_list=None):
+def check_k8s_version_recency(my_version: K8sVersion, cve_version_list=None) -> bool:
     """Check a given K8s cluster version against the list of released versions in order to find out, if the version
     is an accepted recent version according to the standard."""
     if cve_version_list is None:
@@ -400,31 +358,31 @@ def check_k8s_version_recency(version, cve_version_list=None):
         "X-GitHub-Api-Version": "2022-11-28"
     }
 
-    # Request the latest 100 version (the next are not needed, since these versions are too old)
-    response = requests.get("https://api.github.com/repos/kubernetes/kubernetes/releases?per_page=100",
-                            headers=github_headers).json()
-
-    for r in response:
-        v = K8sVersionInfo.extract_version(r['tag_name'].split("-")[0])
-        v.date = datetime.datetime.strptime(r['published_at'], '%Y-%m-%dT%H:%M:%SZ')
+    # Request the latest 100 releases (the next are not needed, since these versions are too old)
+    releases_data = requests.get(
+        "https://api.github.com/repos/kubernetes/kubernetes/releases?per_page=100",
+        headers=github_headers,
+    ).json()
 
-        if r['draft'] or r['prerelease']:
+    for release_data in releases_data:
+        if release_data['draft'] or release_data['prerelease']:
             continue
 
+        release = parse_github_release_data(release_data)
+
         # Check if the version is recent
-        if v.minor >= version.minor:
-            if diff_months(v.date, datetime.datetime.now()) >= MINOR_VERSION_CADENCE_MONTHS:
+        if release.version.minor >= my_version.minor:
+            if release.age > MINOR_VERSION_CADENCE:
                 return False
 
-        if version.check_for_version(major=v.major, minor=v.minor) and version.patch < v.patch:
-            if diff_weeks(datetime.datetime.now(), v.date) >= PATCH_VERSION_CADENCE_WEEKS:
+        if my_version.is_same_minor_version(release.version) and my_version.patch < release.version.patch:
+            if release.age > PATCH_VERSION_CADENCE:
                 return False
 
-            if v in cve_version_list and \
-               diff_days(datetime.datetime.now(), v.date) >= CVE_VERSION_CADENCE_DAYS:
+            if release.version in cve_version_list and release.age > CVE_VERSION_CADENCE:
                 return False
 
-        if v.minor == (version.minor + 1) and v.patch == 0:
+        if release.version.minor == (my_version.minor + 1) and release.version.patch == 0:
             break
 
     return True
@@ -441,25 +399,25 @@ async def main(argv):
 
     connector = aiohttp.TCPConnector(limit=5)
     async with aiohttp.ClientSession(connector=connector) as session:
-        cve_versions = await collect_cve_versions(session)
-    cluster_version, cluster_name = await get_k8s_cluster_version(config.kubeconfig)
+        cve_affected_ranges = await collect_cve_versions(session)
+    cluster = await get_k8s_cluster_info(config.kubeconfig)
 
-    if check_k8s_version_recency(cluster_version, cve_versions):
+    if check_k8s_version_recency(cluster.version, cve_affected_ranges):
         logger.info("The K8s cluster version %s of cluster '%s' is still in the recency time window." %
-                    (str(cluster_version), cluster_name))
+                    (str(cluster.version), cluster.name))
         return 0
 
-    for cvev in cve_versions:
+    for affected_range in cve_affected_ranges:
         try:
-            if cvev.is_version_affected(cluster_version):
+            if cluster.version in affected_range:
                 logger.error("The K8s cluster version %s of cluster '%s' is an outdated version "
-                             "with a possible CRITICAL CVE." % (str(cluster_version), cluster_name))
+                             "with a possible CRITICAL CVE." % (str(cluster.version), cluster.name))
                 return 3
         except TypeError as e:
             logger.error(f"An error occurred during CVE check: {e}")
 
-    logger.error("The K8s cluster version %s of cluster '%s' is outdated according to the Standard." %
-                 (str(cluster_version), cluster_name))
+    logger.error("The K8s cluster version %s of cluster '%s' is outdated according to the standard." %
+                 (str(cluster.version), cluster.name))
     return 2
 
 
diff --git a/Tests/test-requirements.in b/Tests/test-requirements.in
new file mode 100644
index 000000000..ab38ca16d
--- /dev/null
+++ b/Tests/test-requirements.in
@@ -0,0 +1,3 @@
+-c requirements.txt
+pytest
+pytest-asyncio
\ No newline at end of file
diff --git a/Tests/test-requirements.txt b/Tests/test-requirements.txt
new file mode 100644
index 000000000..433a23e5c
--- /dev/null
+++ b/Tests/test-requirements.txt
@@ -0,0 +1,22 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile test-requirements.in
+#
+exceptiongroup==1.2.0
+    # via pytest
+iniconfig==2.0.0
+    # via pytest
+packaging==23.2
+    # via pytest
+pluggy==1.4.0
+    # via pytest
+pytest==8.0.2
+    # via
+    #   -r test-requirements.in
+    #   pytest-asyncio
+pytest-asyncio==0.23.5
+    # via -r test-requirements.in
+tomli==2.0.1
+    # via pytest

From 902f2df3072a743501da8969e217bf12734067bb Mon Sep 17 00:00:00 2001
From: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
Date: Wed, 28 Feb 2024 17:36:06 +0100
Subject: [PATCH 03/13] Add work-in-progress support for v2 of the standard

Signed-off-by: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
---
 .../k8s-version-policy/k8s_version_policy.py  | 62 ++++++++++++-------
 1 file changed, 39 insertions(+), 23 deletions(-)

diff --git a/Tests/kaas/k8s-version-policy/k8s_version_policy.py b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
index 02ed8f8c0..fb45937fa 100755
--- a/Tests/kaas/k8s-version-policy/k8s_version_policy.py
+++ b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
@@ -153,8 +153,9 @@ class K8sVersion:
     minor: int
     patch: int = 0
 
-    def is_same_minor_version(self, other):
-        return self.major == other.major and self.minor == other.minor
+    def branch(self):
+        """Return a tuple of only the major and minor version."""
+        return (self.major, self.minor)
 
     def __str__(self):
         return f"{self.major}.{self.minor}.{self.patch}"
@@ -347,7 +348,7 @@ async def get_k8s_cluster_info(kubeconfig, context=None) -> ClusterInfo:
         return ClusterInfo(version, cluster_config.current_context['name'])
 
 
-def check_k8s_version_recency(my_version: K8sVersion, cve_version_list=None) -> bool:
+def check_k8s_version_recency(my_version: K8sVersion, cve_version_list=None, allow_older=False) -> bool:
     """Check a given K8s cluster version against the list of released versions in order to find out, if the version
     is an accepted recent version according to the standard."""
     if cve_version_list is None:
@@ -370,12 +371,13 @@ def check_k8s_version_recency(my_version: K8sVersion, cve_version_list=None) ->
 
         release = parse_github_release_data(release_data)
 
-        # Check if the version is recent
-        if release.version.minor >= my_version.minor:
+        # Check if the minor version is recent, but allow older versions if requested
+        # FIXME: this assumes k8s stays in 1.x version schema :(
+        if release.version.minor >= my_version.minor and not allow_older:
             if release.age > MINOR_VERSION_CADENCE:
                 return False
 
-        if my_version.is_same_minor_version(release.version) and my_version.patch < release.version.patch:
+        if my_version.branch() == release.version.branch() and my_version.patch < release.version.patch:
             if release.age > PATCH_VERSION_CADENCE:
                 return False
 
@@ -400,25 +402,39 @@ async def main(argv):
     connector = aiohttp.TCPConnector(limit=5)
     async with aiohttp.ClientSession(connector=connector) as session:
         cve_affected_ranges = await collect_cve_versions(session)
-    cluster = await get_k8s_cluster_info(config.kubeconfig)
 
-    if check_k8s_version_recency(cluster.version, cve_affected_ranges):
-        logger.info("The K8s cluster version %s of cluster '%s' is still in the recency time window." %
-                    (str(cluster.version), cluster.name))
-        return 0
+    contexts = ["stable", "oldstable", "oldoldstable"]
+    branches = set()
+
+    for context in contexts:
+        cluster = await get_k8s_cluster_info(config.kubeconfig, context)
+        branches.add(cluster.version.branch())
+        allow_older = context == contexts[0]
+
+        if check_k8s_version_recency(cluster.version, cve_affected_ranges, allow_older):
+            logger.info("The K8s cluster version %s of cluster '%s' is still in the recency time window." %
+                        (str(cluster.version), cluster.name))
+        else:
+            logger.error("The K8s cluster version %s of cluster '%s' is outdated according to the standard." %
+                        (str(cluster.version), cluster.name))
+            return 2
+
+        for affected_range in cve_affected_ranges:
+            try:
+                if cluster.version in affected_range:
+                    logger.error("The K8s cluster version %s of cluster '%s' is an outdated version "
+                                "with a possible CRITICAL CVE." % (str(cluster.version), cluster.name))
+                    return 3
+            except TypeError as e:
+                logger.error(f"An error occurred during CVE check: {e}")
+    
+    if len(branches) < 3:
+        # TODO
+        logger.error("support period")
+        return 4
+
+    return 0
 
-    for affected_range in cve_affected_ranges:
-        try:
-            if cluster.version in affected_range:
-                logger.error("The K8s cluster version %s of cluster '%s' is an outdated version "
-                             "with a possible CRITICAL CVE." % (str(cluster.version), cluster.name))
-                return 3
-        except TypeError as e:
-            logger.error(f"An error occurred during CVE check: {e}")
-
-    logger.error("The K8s cluster version %s of cluster '%s' is outdated according to the standard." %
-                 (str(cluster.version), cluster.name))
-    return 2
 
 
 if __name__ == "__main__":

From 38756be098b0817b3d2d1a871d3c39883945dcc8 Mon Sep 17 00:00:00 2001
From: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
Date: Thu, 29 Feb 2024 07:32:10 +0100
Subject: [PATCH 04/13] Bugfix

Signed-off-by: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
---
 Tests/kaas/k8s-version-policy/k8s_version_policy.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Tests/kaas/k8s-version-policy/k8s_version_policy.py b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
index fb45937fa..60554defa 100755
--- a/Tests/kaas/k8s-version-policy/k8s_version_policy.py
+++ b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
@@ -409,7 +409,8 @@ async def main(argv):
     for context in contexts:
         cluster = await get_k8s_cluster_info(config.kubeconfig, context)
         branches.add(cluster.version.branch())
-        allow_older = context == contexts[0]
+        # allow older k8s branches, but not for the first context (stable)
+        allow_older = context != contexts[0]
 
         if check_k8s_version_recency(cluster.version, cve_affected_ranges, allow_older):
             logger.info("The K8s cluster version %s of cluster '%s' is still in the recency time window." %

From eb5fe88c92c7386963dd7eb5db959b66a3653457 Mon Sep 17 00:00:00 2001
From: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
Date: Thu, 29 Feb 2024 07:36:23 +0100
Subject: [PATCH 05/13] Format for flake8 and fix the logging statements

Signed-off-by: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
---
 .../k8s-version-policy/config.yaml.template   |  2 +-
 .../k8s-version-policy/k8s_version_policy.py  | 26 ++++++++++++-------
 Tests/test-requirements.in                    |  2 +-
 3 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/Tests/kaas/k8s-version-policy/config.yaml.template b/Tests/kaas/k8s-version-policy/config.yaml.template
index e7a247c8d..cfe6a867c 100644
--- a/Tests/kaas/k8s-version-policy/config.yaml.template
+++ b/Tests/kaas/k8s-version-policy/config.yaml.template
@@ -21,4 +21,4 @@ logging:
   loggers:
     k8s_version_policy:
       handlers: [console, file]
-      propagate: no
\ No newline at end of file
+      propagate: no
diff --git a/Tests/kaas/k8s-version-policy/k8s_version_policy.py b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
index 60554defa..f25b9d471 100755
--- a/Tests/kaas/k8s-version-policy/k8s_version_policy.py
+++ b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
@@ -413,22 +413,31 @@ async def main(argv):
         allow_older = context != contexts[0]
 
         if check_k8s_version_recency(cluster.version, cve_affected_ranges, allow_older):
-            logger.info("The K8s cluster version %s of cluster '%s' is still in the recency time window." %
-                        (str(cluster.version), cluster.name))
+            logger.info(
+                "The K8s cluster version %s of cluster '%s' is still in the recency time window.",
+                cluster.version,
+                cluster.name,
+            )
         else:
-            logger.error("The K8s cluster version %s of cluster '%s' is outdated according to the standard." %
-                        (str(cluster.version), cluster.name))
+            logger.error(
+                "The K8s cluster version %s of cluster '%s' is outdated according to the standard.",
+                cluster.version,
+                cluster.name,
+            )
             return 2
 
         for affected_range in cve_affected_ranges:
             try:
                 if cluster.version in affected_range:
-                    logger.error("The K8s cluster version %s of cluster '%s' is an outdated version "
-                                "with a possible CRITICAL CVE." % (str(cluster.version), cluster.name))
+                    logger.error(
+                        "The K8s cluster version %s of cluster '%s' is an outdated version with a possible CRITICAL CVE.",
+                        cluster.version,
+                        cluster.name,
+                    )
                     return 3
             except TypeError as e:
-                logger.error(f"An error occurred during CVE check: {e}")
-    
+                logger.error("An error occurred during CVE check: %s", e)
+
     if len(branches) < 3:
         # TODO
         logger.error("support period")
@@ -437,7 +446,6 @@ async def main(argv):
     return 0
 
 
-
 if __name__ == "__main__":
     return_code = asyncio.run(main(sys.argv[1:]))
     sys.exit(return_code)
diff --git a/Tests/test-requirements.in b/Tests/test-requirements.in
index ab38ca16d..5ac7ac7d3 100644
--- a/Tests/test-requirements.in
+++ b/Tests/test-requirements.in
@@ -1,3 +1,3 @@
 -c requirements.txt
 pytest
-pytest-asyncio
\ No newline at end of file
+pytest-asyncio

From 4e839b225b4194076e79edcb5720b7bc9419bc1f Mon Sep 17 00:00:00 2001
From: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
Date: Thu, 29 Feb 2024 07:37:50 +0100
Subject: [PATCH 06/13] Use set where it is suitable

Signed-off-by: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
---
 .../k8s-version-policy/k8s_version_policy.py    | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/Tests/kaas/k8s-version-policy/k8s_version_policy.py b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
index f25b9d471..3f2b06953 100755
--- a/Tests/kaas/k8s-version-policy/k8s_version_policy.py
+++ b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
@@ -147,7 +147,7 @@ def initialize_config(config):
     return config
 
 
-@dataclass(order=True)
+@dataclass(frozen=True, eq=True, order=True)
 class K8sVersion:
     major: int
     minor: int
@@ -170,7 +170,7 @@ def parse_version(version_str: str) -> K8sVersion:
         raise ValueError(f"Unrecognized version format: {version_str}")
 
 
-@dataclass
+@dataclass(frozen=True)
 class K8sRelease:
     version: K8sVersion
     released_at: datetime
@@ -189,7 +189,7 @@ def parse_github_release_data(release_data: dict) -> K8sRelease:
     return K8sRelease(version, released_at)
 
 
-@dataclass
+@dataclass(frozen=True, eq=True)
 class VersionRange:
     """Version range with an lower and upper bound."""
 
@@ -281,13 +281,13 @@ def is_high_severity(cve_metrics: list) -> bool:
     )
 
 
-async def collect_cve_versions(session: aiohttp.ClientSession):
+async def collect_cve_versions(session: aiohttp.ClientSession) -> set:
     """Get all relevant CVE versions, that are relevant for the test according to the severity
     dictated by the Standard.
     """
 
     # CVE fix versions
-    cfvs = list()
+    cfvs = set()
 
     # Request latest version
     async with session.get(
@@ -327,12 +327,7 @@ async def collect_cve_versions(session: aiohttp.ClientSession):
                 for version_info in aff['versions']
                 if version_info['status'] == "affected"
             ]
-            for cvev in affected_kubernetes_versions:
-                try:
-                    if cvev not in cfvs:
-                        cfvs.append(cvev)
-                except TypeError:
-                    pass
+            cfvs.update(affected_kubernetes_versions)
 
     return cfvs
 

From 253fb973f52622f3a003ef8bb6acd02c38e4049a Mon Sep 17 00:00:00 2001
From: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
Date: Thu, 29 Feb 2024 07:49:35 +0100
Subject: [PATCH 07/13] Update the help string and doc comments

Signed-off-by: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
---
 .../k8s-version-policy/k8s_version_policy.py  | 31 ++++++++++++-------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/Tests/kaas/k8s-version-policy/k8s_version_policy.py b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
index 3f2b06953..adb4776e3 100755
--- a/Tests/kaas/k8s-version-policy/k8s_version_policy.py
+++ b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
@@ -2,22 +2,28 @@
 # vim: set ts=4 sw=4 et:
 #
 """
-K8s Version Recency Checker
+K8s Version Policy Checker (scs-v0210-v2)
 https://github.com/SovereignCloudStack/standards
 
 Return codes:
-0: Version is inside the recency window
+0: All versions are inside the recency window
 1: Error during script execution
-2: Version isn't inside the recency windows anymore
-3: Version used contains a critical CVE
-
-One or more K8s clusters are checked by providing their kubeconfigs.
-It is determined, if the version on these clusters is still inside
-the recency window, which is determined by the Standard to be 4 months
-for minor versions and 1 week for patch versions. An exception are
-versions with critical CVEs, which should be replaced on a shorter notice.
+2: A cluster version isn't inside the recency windows anymore
+3: A cluster version used contains a critical CVE
+4: Support for a non-EOL Kubernetes version is missing
+
+The K8s clusters provided in a kubeconfig are checked. The kubeconfig
+must provide connection details for the clusters to be tested via
+the contexts "stable", "oldstable", "oldoldstable" and "oldoldoldstable",
+depending on how many upstream K8s releases are currently supported.
+It is determined if the version on these clusters is still inside
+the recency window, which is determined by the standard to be 4 months
+for minor versions (for the stable cluster) and 1 week for patch versions.
+An exception are versions with critical CVEs, which should be replaced on
+a shorter notice.
 
 (c) Hannes Baum <hannes.baum@cloudandheat.com>, 6/2023
+(c) Martin Morgenstern <martin.morgenstern@cloudandheat.com>, 2/2024
 License: CC-BY-SA 4.0
 """
 
@@ -81,12 +87,13 @@ class Config:
 
 def print_usage():
     print("""
-K8s Version Recency Compliance Check
+K8s Version Policy Compliance Check
 
 Usage: k8s_version_policy.py [-h] [-c|--config PATH/TO/CONFIG] -k|--kubeconfig PATH/TO/KUBECONFIG
 
-The K8s version recency check returns 0 if the version of the tested cluster is still acceptable, otherwise
+The K8s version policy check returns 0 if the versions of the tested clusters are still acceptable, otherwise
 it returns 2 for an out-of date version or 3 if the used version should be updated due to a highly critical CVE.
+It returns 4 if a supported upstream K8s release is missing.
 
     -c/--config PATH/TO/CONFIG         - Path to the config file of the test script
     -k/--kubeconfig PATH/TO/KUBECONFIG - Path to the kubeconfig of the server we want to check

From 10270a3349d01a3f43b8275f9095102d5b969b05 Mon Sep 17 00:00:00 2001
From: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
Date: Thu, 29 Feb 2024 09:11:20 +0100
Subject: [PATCH 08/13] Add proper check if supported K8s upstream branches are
 provided

Signed-off-by: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
---
 .../kaas/k8s-version-policy/k8s-eol-data.yml  | 12 +--
 .../k8s-version-policy/k8s_version_policy.py  | 92 +++++++++++++++++--
 2 files changed, 88 insertions(+), 16 deletions(-)

diff --git a/Tests/kaas/k8s-version-policy/k8s-eol-data.yml b/Tests/kaas/k8s-version-policy/k8s-eol-data.yml
index 500e53ae0..4efe0634b 100644
--- a/Tests/kaas/k8s-version-policy/k8s-eol-data.yml
+++ b/Tests/kaas/k8s-version-policy/k8s-eol-data.yml
@@ -1,12 +1,12 @@
 # https://kubernetes.io/releases/patch-releases/#detailed-release-history-for-active-branches
-# Please keep this list sorted descending by version.
-- version: '1.29'
+
+- branch: '1.29'
   end-of-life: '2025-02-28'
-- version: '1.28'
+- branch: '1.28'
   end-of-life: '2024-10-28'
-- version: '1.27'
+- branch: '1.27'
   end-of-life: '2024-06-28'
-- version: '1.26'
+- branch: '1.26'
   end-of-life: '2024-02-28'
-- version: '1.25'
+- branch: '1.25'
   end-of-life: '2023-10-28'
diff --git a/Tests/kaas/k8s-version-policy/k8s_version_policy.py b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
index adb4776e3..6e5c21619 100755
--- a/Tests/kaas/k8s-version-policy/k8s_version_policy.py
+++ b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
@@ -29,6 +29,7 @@
 
 from dataclasses import dataclass
 from datetime import datetime, timedelta
+from pathlib import Path
 import aiohttp
 import asyncio
 import contextlib
@@ -47,6 +48,8 @@
 CVE_VERSION_CADENCE = timedelta(days=3)
 CVE_SEVERITY = 8  # CRITICAL
 
+HERE = Path(__file__).parent
+
 logging_config = {
     "level": "INFO",
     "version": 1,
@@ -160,9 +163,10 @@ class K8sVersion:
     minor: int
     patch: int = 0
 
+    @property
     def branch(self):
-        """Return a tuple of only the major and minor version."""
-        return (self.major, self.minor)
+        """Get the branch of this version, i.e., the version w/o patch level."""
+        return K8sBranch(self.major, self.minor)
 
     def __str__(self):
         return f"{self.major}.{self.minor}.{self.patch}"
@@ -177,6 +181,35 @@ def parse_version(version_str: str) -> K8sVersion:
         raise ValueError(f"Unrecognized version format: {version_str}")
 
 
+@dataclass(frozen=True, eq=True, order=True)
+class K8sBranch:
+    """Identifies a release branch of K8s just by major and minor version."""
+
+    major: int
+    minor: int
+
+    def previous(self):
+        if self.minor == 0:
+            # FIXME: this is ugly
+            return self
+        return K8sBranch(self.major, self.minor - 1)
+
+    def __str__(self):
+        return f"{self.major}.{self.minor}"
+
+
+@dataclass(frozen=True)
+class K8sBranchInfo:
+    branch: K8sBranch
+    eol: datetime
+
+    def is_supported(self) -> bool:
+        return datetime.now() < self.eol
+
+    def is_eol(self) -> bool:
+        return not self.is_supported()
+
+
 @dataclass(frozen=True)
 class K8sRelease:
     version: K8sVersion
@@ -198,7 +231,7 @@ def parse_github_release_data(release_data: dict) -> K8sRelease:
 
 @dataclass(frozen=True, eq=True)
 class VersionRange:
-    """Version range with an lower and upper bound."""
+    """Version range with a lower and upper bound."""
 
     # First version with the CVE; this value will be set if either an affected
     # version is directly set in a CVE dataset or if the CVE dataset is in a
@@ -379,7 +412,7 @@ def check_k8s_version_recency(my_version: K8sVersion, cve_version_list=None, all
             if release.age > MINOR_VERSION_CADENCE:
                 return False
 
-        if my_version.branch() == release.version.branch() and my_version.patch < release.version.patch:
+        if my_version.branch == release.version.branch and my_version.patch < release.version.patch:
             if release.age > PATCH_VERSION_CADENCE:
                 return False
 
@@ -392,6 +425,20 @@ def check_k8s_version_recency(my_version: K8sVersion, cve_version_list=None, all
     return True
 
 
+def parse_branch_info(data: dict) -> K8sBranchInfo:
+    major, minor = data["branch"].split(".")
+    branch = K8sBranch(int(major), int(minor))
+    eol_date = datetime.strptime(data["end-of-life"], "%Y-%m-%d")
+    return K8sBranchInfo(branch, eol_date)
+
+
+def read_supported_k8s_branches(eol_data_path: Path) -> dict[K8sBranch, K8sBranchInfo]:
+    with open(eol_data_path) as stream:
+        data = yaml.load(stream, Loader=yaml.FullLoader)
+    infos = [parse_branch_info(item) for item in data]
+    return {info.branch: info for info in infos}
+
+
 async def main(argv):
     try:
         config = initialize_config(parse_arguments(argv))
@@ -405,12 +452,22 @@ async def main(argv):
     async with aiohttp.ClientSession(connector=connector) as session:
         cve_affected_ranges = await collect_cve_versions(session)
 
-    contexts = ["stable", "oldstable", "oldoldstable"]
-    branches = set()
+    contexts = ["stable", "oldstable", "oldoldstable", "oldoldoldstable"]
+    branch_infos = read_supported_k8s_branches(Path(HERE, "k8s-eol-data.yml"))
+    supported_branches = {
+        branch
+        for branch, branch_info
+        in branch_infos.items()
+        if branch_info.is_supported()
+    }
+    seen_branches = set()
 
     for context in contexts:
+        logger.info("Checking cluster of kubeconfig context '%s'.", context)
         cluster = await get_k8s_cluster_info(config.kubeconfig, context)
-        branches.add(cluster.version.branch())
+        cluster_branch = cluster.version.branch
+        seen_branches.add(cluster_branch)
+
         # allow older k8s branches, but not for the first context (stable)
         allow_older = context != contexts[0]
 
@@ -440,9 +497,24 @@ async def main(argv):
             except TypeError as e:
                 logger.error("An error occurred during CVE check: %s", e)
 
-    if len(branches) < 3:
-        # TODO
-        logger.error("support period")
+        # this is also a bit ugly
+        if context == "oldoldstable" and branch_infos[cluster_branch.previous()].is_eol():
+            logger.info("Skipping the next context because the cluster it should reference is already EOL.")
+            break
+
+    # Now check if we saw all upstream supported K8s branches. Keep in mind
+    # that providers have a cadence time to update the "stable" context to the
+    # newest K8s release branch.
+    expected_branches = set(supported_branches)
+    newest_branch = max(supported_branches)
+    newest_branch_seen = max(seen_branches)
+    if newest_branch != newest_branch_seen:
+        expected_branches.remove(newest_branch)
+
+    if seen_branches != expected_branches:
+        missing = expected_branches - seen_branches
+        listing = " ".join(f"{branch}" for branch in missing)
+        logger.error("The following upstream branches should be supported but were missing: %s", listing)
         return 4
 
     return 0

From 33320d3a236f7eec15b1ad685263ff7120c44380 Mon Sep 17 00:00:00 2001
From: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
Date: Thu, 29 Feb 2024 09:12:29 +0100
Subject: [PATCH 09/13] Improve error handling and logging

Signed-off-by: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
---
 Tests/kaas/k8s-version-policy/k8s_version_policy.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/Tests/kaas/k8s-version-policy/k8s_version_policy.py b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
index 6e5c21619..a8d2fc88f 100755
--- a/Tests/kaas/k8s-version-policy/k8s_version_policy.py
+++ b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
@@ -29,6 +29,7 @@
 
 from dataclasses import dataclass
 from datetime import datetime, timedelta
+from kubernetes_asyncio.config import ConfigException as KubeConfigException
 from pathlib import Path
 import aiohttp
 import asyncio
@@ -464,7 +465,11 @@ async def main(argv):
 
     for context in contexts:
         logger.info("Checking cluster of kubeconfig context '%s'.", context)
-        cluster = await get_k8s_cluster_info(config.kubeconfig, context)
+        try:
+            cluster = await get_k8s_cluster_info(config.kubeconfig, context)
+        except KubeConfigException as e:
+            logger.error("There was an error while connecting to the cluster: %s", e)
+            return 1
         cluster_branch = cluster.version.branch
         seen_branches.add(cluster_branch)
 
@@ -496,6 +501,7 @@ async def main(argv):
                     return 3
             except TypeError as e:
                 logger.error("An error occurred during CVE check: %s", e)
+                return 1
 
         # this is also a bit ugly
         if context == "oldoldstable" and branch_infos[cluster_branch.previous()].is_eol():

From 4400b9a0d21c78ef0f9674448fbf1b90a2624174 Mon Sep 17 00:00:00 2001
From: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
Date: Thu, 29 Feb 2024 09:27:52 +0100
Subject: [PATCH 10/13] Fix faulty calculation and display of missing versions

Signed-off-by: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
---
 Tests/kaas/k8s-version-policy/k8s_version_policy.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Tests/kaas/k8s-version-policy/k8s_version_policy.py b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
index a8d2fc88f..84a24c0f0 100755
--- a/Tests/kaas/k8s-version-policy/k8s_version_policy.py
+++ b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
@@ -503,22 +503,22 @@ async def main(argv):
                 logger.error("An error occurred during CVE check: %s", e)
                 return 1
 
-        # this is also a bit ugly
-        if context == "oldoldstable" and branch_infos[cluster_branch.previous()].is_eol():
+        if branch_infos[cluster_branch.previous()].is_eol():
             logger.info("Skipping the next context because the cluster it should reference is already EOL.")
             break
 
     # Now check if we saw all upstream supported K8s branches. Keep in mind
     # that providers have a cadence time to update the "stable" context to the
-    # newest K8s release branch.
+    # newest K8s release branch. The corresponding window was already checked
+    # above.
     expected_branches = set(supported_branches)
     newest_branch = max(supported_branches)
     newest_branch_seen = max(seen_branches)
     if newest_branch != newest_branch_seen:
         expected_branches.remove(newest_branch)
 
-    if seen_branches != expected_branches:
-        missing = expected_branches - seen_branches
+    missing = expected_branches - seen_branches
+    if missing:
         listing = " ".join(f"{branch}" for branch in missing)
         logger.error("The following upstream branches should be supported but were missing: %s", listing)
         return 4

From fae25c6ae8b4a90de0f39ce644227c74512d6763 Mon Sep 17 00:00:00 2001
From: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
Date: Mon, 4 Mar 2024 15:50:20 +0100
Subject: [PATCH 11/13] Incorporate code improvements suggested by reviewers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Thanks @cah-hbaum and @mbuechse!

Co-authored-by: Hannes Baum <hannes.baum@cloudandheat.com>
Co-authored-by: Matthias Büchse <matthias.buechse@cloudandheat.com>
Signed-off-by: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
---
 Tests/kaas/k8s-version-policy/k8s_version_policy.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Tests/kaas/k8s-version-policy/k8s_version_policy.py b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
index 84a24c0f0..4e5e3abf6 100755
--- a/Tests/kaas/k8s-version-policy/k8s_version_policy.py
+++ b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
@@ -102,7 +102,7 @@ def print_usage():
     -c/--config PATH/TO/CONFIG         - Path to the config file of the test script
     -k/--kubeconfig PATH/TO/KUBECONFIG - Path to the kubeconfig of the server we want to check
     -h                                 - Output help
-    """)
+""")
 
 
 def parse_arguments(argv):
@@ -110,7 +110,7 @@ def parse_arguments(argv):
     config = Config()
 
     try:
-        opts, args = getopt.gnu_getopt(argv, "c:k:h", ["config", "kubeconfig", "help"])
+        opts, args = getopt.gnu_getopt(argv, "c:k:h", ["config=", "kubeconfig=", "help"])
     except getopt.GetoptError:
         raise ConfigException
 

From dd3f27c510929b34e4282ff62caa30344785299b Mon Sep 17 00:00:00 2001
From: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
Date: Mon, 4 Mar 2024 17:01:30 +0100
Subject: [PATCH 12/13] Try to collect errors for all contexts instead of
 failing early

Instead of failing immediately, try to test *all* contexts/clusters before
exiting the script. Only exit early in critical cases.

Signed-off-by: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
---
 .../k8s-version-policy/k8s_version_policy.py  | 101 ++++++++++--------
 1 file changed, 56 insertions(+), 45 deletions(-)

diff --git a/Tests/kaas/k8s-version-policy/k8s_version_policy.py b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
index 4e5e3abf6..706520c54 100755
--- a/Tests/kaas/k8s-version-policy/k8s_version_policy.py
+++ b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
@@ -5,12 +5,13 @@
 K8s Version Policy Checker (scs-v0210-v2)
 https://github.com/SovereignCloudStack/standards
 
-Return codes:
-0: All versions are inside the recency window
-1: Error during script execution
-2: A cluster version isn't inside the recency windows anymore
-3: A cluster version used contains a critical CVE
-4: Support for a non-EOL Kubernetes version is missing
+Return code is 0 precisely when it could be verified that the standard is satisfied.
+Otherwise the return code is the number of errors that occurred (up to 127 due to OS
+restrictions); for further information, see the log messages on various channels:
+    CRITICAL  for problems preventing the test to complete,
+    ERROR     for violations of requirements,
+    INFO      for violations of recommendations,
+    DEBUG     for background information and problems that don't hinder the test.
 
 The K8s clusters provided in a kubeconfig are checked. The kubeconfig
 must provide connection details for the clusters to be tested via
@@ -27,9 +28,9 @@
 License: CC-BY-SA 4.0
 """
 
+from collections import Counter
 from dataclasses import dataclass
 from datetime import datetime, timedelta
-from kubernetes_asyncio.config import ConfigException as KubeConfigException
 from pathlib import Path
 import aiohttp
 import asyncio
@@ -75,6 +76,15 @@
 logger = logging.getLogger(__name__)
 
 
+class CountingHandler(logging.Handler):
+    def __init__(self, level=logging.NOTSET):
+        super().__init__(level=level)
+        self.bylevel = Counter()
+
+    def handle(self, record):
+        self.bylevel[record.levelno] += 1
+
+
 class ConfigException(BaseException):
     """Exception raised in a configuration error occurs"""
 
@@ -444,11 +454,13 @@ async def main(argv):
     try:
         config = initialize_config(parse_arguments(argv))
     except (OSError, ConfigException, HelpException) as e:
-        if hasattr(e, 'message'):
-            logger.error(e.message)
+        logger.critical("%s", e)
         print_usage()
         return 1
 
+    counting_handler = CountingHandler(level=logging.INFO)
+    logger.addHandler(counting_handler)
+
     connector = aiohttp.TCPConnector(limit=5)
     async with aiohttp.ClientSession(connector=connector) as session:
         cve_affected_ranges = await collect_cve_versions(session)
@@ -463,49 +475,44 @@ async def main(argv):
     }
     seen_branches = set()
 
-    for context in contexts:
-        logger.info("Checking cluster of kubeconfig context '%s'.", context)
-        try:
+    try:
+        for context in contexts:
+            logger.info("Checking cluster of kubeconfig context '%s'.", context)
             cluster = await get_k8s_cluster_info(config.kubeconfig, context)
-        except KubeConfigException as e:
-            logger.error("There was an error while connecting to the cluster: %s", e)
-            return 1
-        cluster_branch = cluster.version.branch
-        seen_branches.add(cluster_branch)
-
-        # allow older k8s branches, but not for the first context (stable)
-        allow_older = context != contexts[0]
-
-        if check_k8s_version_recency(cluster.version, cve_affected_ranges, allow_older):
-            logger.info(
-                "The K8s cluster version %s of cluster '%s' is still in the recency time window.",
-                cluster.version,
-                cluster.name,
-            )
-        else:
-            logger.error(
-                "The K8s cluster version %s of cluster '%s' is outdated according to the standard.",
-                cluster.version,
-                cluster.name,
-            )
-            return 2
+            cluster_branch = cluster.version.branch
+            seen_branches.add(cluster_branch)
+
+            # allow older k8s branches, but not for the first context (stable)
+            allow_older = context != contexts[0]
+
+            if check_k8s_version_recency(cluster.version, cve_affected_ranges, allow_older):
+                logger.info(
+                    "The K8s cluster version %s of cluster '%s' is still in the recency time window.",
+                    cluster.version,
+                    cluster.name,
+                )
+            else:
+                logger.error(
+                    "The K8s cluster version %s of cluster '%s' is outdated according to the standard.",
+                    cluster.version,
+                    cluster.name,
+                )
 
-        for affected_range in cve_affected_ranges:
-            try:
+            for affected_range in cve_affected_ranges:
                 if cluster.version in affected_range:
                     logger.error(
                         "The K8s cluster version %s of cluster '%s' is an outdated version with a possible CRITICAL CVE.",
                         cluster.version,
                         cluster.name,
                     )
-                    return 3
-            except TypeError as e:
-                logger.error("An error occurred during CVE check: %s", e)
-                return 1
 
-        if branch_infos[cluster_branch.previous()].is_eol():
-            logger.info("Skipping the next context because the cluster it should reference is already EOL.")
-            break
+            if branch_infos[cluster_branch.previous()].is_eol():
+                logger.info("Skipping the next context because the cluster it should reference is already EOL.")
+                break
+    except BaseException as e:
+        logger.critical("%s", e)
+        logger.debug("Exception info", exc_info=True)
+        return 1
 
     # Now check if we saw all upstream supported K8s branches. Keep in mind
     # that providers have a cadence time to update the "stable" context to the
@@ -521,9 +528,13 @@ async def main(argv):
     if missing:
         listing = " ".join(f"{branch}" for branch in missing)
         logger.error("The following upstream branches should be supported but were missing: %s", listing)
-        return 4
 
-    return 0
+    c = counting_handler.bylevel
+    logger.debug(
+        "Total error / warning: "
+        f"{c[logging.ERROR]} / {c[logging.WARNING]}"
+    )
+    return min(127, c[logging.ERROR])  # cap at 127 due to OS restrictions
 
 
 if __name__ == "__main__":

From 45983ba2b5f5b4035f237358a8292d195c6c4c78 Mon Sep 17 00:00:00 2001
From: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
Date: Tue, 5 Mar 2024 10:21:10 +0100
Subject: [PATCH 13/13] Warn or bail out if EOL data is outdated

Signed-off-by: Martin Morgenstern <martin.morgenstern@cloudandheat.com>
---
 .../k8s-version-policy/k8s_version_policy.py  | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/Tests/kaas/k8s-version-policy/k8s_version_policy.py b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
index 706520c54..490cf74d1 100755
--- a/Tests/kaas/k8s-version-policy/k8s_version_policy.py
+++ b/Tests/kaas/k8s-version-policy/k8s_version_policy.py
@@ -51,6 +51,7 @@
 CVE_SEVERITY = 8  # CRITICAL
 
 HERE = Path(__file__).parent
+EOLDATA_FILE = "k8s-eol-data.yml"
 
 logging_config = {
     "level": "INFO",
@@ -461,18 +462,24 @@ async def main(argv):
     counting_handler = CountingHandler(level=logging.INFO)
     logger.addHandler(counting_handler)
 
-    connector = aiohttp.TCPConnector(limit=5)
-    async with aiohttp.ClientSession(connector=connector) as session:
-        cve_affected_ranges = await collect_cve_versions(session)
-
-    contexts = ["stable", "oldstable", "oldoldstable", "oldoldoldstable"]
-    branch_infos = read_supported_k8s_branches(Path(HERE, "k8s-eol-data.yml"))
+    branch_infos = read_supported_k8s_branches(Path(HERE, EOLDATA_FILE))
     supported_branches = {
         branch
         for branch, branch_info
         in branch_infos.items()
         if branch_info.is_supported()
     }
+    if len(supported_branches) < 3:
+        logger.warning("The EOL data in %s isn't up-to-date.", EOLDATA_FILE)
+    if len(supported_branches) < 2:
+        logger.critical("The EOL data in %s is outdated and we cannot reliably run this script.", EOLDATA_FILE)
+        return 1
+
+    connector = aiohttp.TCPConnector(limit=5)
+    async with aiohttp.ClientSession(connector=connector) as session:
+        cve_affected_ranges = await collect_cve_versions(session)
+
+    contexts = ["stable", "oldstable", "oldoldstable", "oldoldoldstable"]
     seen_branches = set()
 
     try: