|
| 1 | +#!/bin/bash |
| 2 | + |
| 3 | +# Copyright 2025 The Kubernetes Authors. |
| 4 | +# |
| 5 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | +# you may not use this file except in compliance with the License. |
| 7 | +# You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, software |
| 12 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | +# See the License for the specific language governing permissions and |
| 15 | +# limitations under the License. |
| 16 | + |
| 17 | +# Parameter set definitions for e2e parameter tests. |
| 18 | +# Each set defines GINKGO_FOCUS, HELM_EXTRA_FLAGS, and optionally other env vars. |
| 19 | +# |
| 20 | +# Sets in PARAM_SETS_ALL (no "special" cluster config needed): |
| 21 | +# standard - Behavioral params (tagging, metrics, logging, storage classes, etc.) |
| 22 | +# other - Volume modification, volume attach limit, and metadata labeler |
| 23 | +# debug - debugLogs=true overrides individual logLevel settings |
| 24 | +# infra - Infrastructure/deployment params (resources, security, strategy, etc.) |
| 25 | +# node-component-only - Deploys only node DaemonSet without controller |
| 26 | +# fips - Builds FIPS image then validates it is deployed |
| 27 | +# |
| 28 | +# Not in PARAM_SETS_ALL: |
| 29 | +# legacy-compat - Legacy CSIDriver + XFS behavior |
| 30 | +# selinux - Needs SELinux-enabled nodes |
| 31 | + |
| 32 | +set -euo pipefail |
| 33 | + |
| 34 | +PARAM_SETS_ALL="standard other debug infra node-component-only fips" |
| 35 | + |
| 36 | +param_set_standard() { |
| 37 | + GINKGO_FOCUS="\[param:(extraCreateMetadata|k8sTagClusterId|extraVolumeTags|controllerMetrics|nodeMetrics|batching|defaultFsType|controllerLoggingFormat|nodeLoggingFormat|controllerLogLevel|nodeLogLevel|provisionerLogLevel|attacherLogLevel|snapshotterLogLevel|resizerLogLevel|nodeDriverRegistrarLogLevel|storageClasses|volumeSnapshotClasses|defaultStorageClass|snapshotterForceEnable|controllerUserAgentExtra|controllerEnablePrometheusAnnotations|nodeEnablePrometheusAnnotations|nodeKubeletPath|nodeTolerateAllTaints|controllerPodDisruptionBudget|provisionerLeaderElection|attacherLeaderElection|resizerLeaderElection|reservedVolumeAttachments|hostNetwork|nodeDisableMutation|nodeTerminationGracePeriod|nodeAllocatableUpdatePeriodSeconds|controllerServiceMonitor|nodeServiceMonitor)\]" |
| 38 | + HELM_EXTRA_FLAGS="--set=controller.extraCreateMetadata=true,controller.k8sTagClusterId=e2e-param-test,controller.extraVolumeTags.TestKey=TestValue,controller.enableMetrics=true,node.enableMetrics=true,controller.batching=true,controller.defaultFsType=xfs,controller.loggingFormat=json,node.loggingFormat=json,controller.logLevel=5,node.logLevel=5,sidecars.provisioner.logLevel=5,sidecars.attacher.logLevel=5,sidecars.snapshotter.logLevel=5,sidecars.resizer.logLevel=5,sidecars.nodeDriverRegistrar.logLevel=5,defaultStorageClass.enabled=true,storageClasses[0].name=test-sc,storageClasses[0].parameters.type=gp3,volumeSnapshotClasses[0].name=test-vsc,volumeSnapshotClasses[0].deletionPolicy=Delete,sidecars.snapshotter.forceEnable=true,controller.userAgentExtra=e2e-test,controller.enablePrometheusAnnotations=true,node.enablePrometheusAnnotations=true,node.kubeletPath=/var/lib/kubelet,node.tolerateAllTaints=true,controller.podDisruptionBudget.enabled=true,sidecars.provisioner.leaderElection.enabled=true,sidecars.attacher.leaderElection.enabled=true,sidecars.resizer.leaderElection.enabled=true,node.reservedVolumeAttachments=2,node.hostNetwork=true,node.serviceAccount.disableMutation=true,node.terminationGracePeriodSeconds=60,nodeAllocatableUpdatePeriodSeconds=30,controller.serviceMonitor.forceEnable=true,node.serviceMonitor.forceEnable=true" |
| 39 | + # Install ServiceMonitor CRD for serviceMonitor tests |
| 40 | + kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml |
| 41 | +} |
| 42 | + |
| 43 | +# other combines volume-modification, volume-attach-limit, and metadata-labeler (no conflicting Helm values) |
| 44 | +param_set_other() { |
| 45 | + GINKGO_FOCUS="\[param:(volumeModification|volumemodifierLogLevel|volumemodifierLeaderElection|volumeAttachLimit|metadataLabeler|metadataLabelerLogLevel|additionalDaemonSets)\]" |
| 46 | + HELM_EXTRA_FLAGS="--set=controller.volumeModificationFeature.enabled=true,sidecars.provisioner.additionalArgs[0]='--feature-gates=VolumeAttributesClass=true',sidecars.resizer.additionalArgs[0]='--feature-gates=VolumeAttributesClass=true',sidecars.volumemodifier.logLevel=5,sidecars.volumemodifier.leaderElection.enabled=false,node.volumeAttachLimit=25,sidecars.metadataLabeler.enabled=true,node.metadataSources='metadata-labeler',sidecars.metadataLabeler.logLevel=5,additionalDaemonSets.extra.volumeAttachLimit=15" |
| 47 | +} |
| 48 | + |
| 49 | +# debugLogs=true overrides individual logLevel settings, so this must be separate from standard |
| 50 | +param_set_debug() { |
| 51 | + GINKGO_FOCUS="\[param:(debugLogs|sdkDebugLog)\]" |
| 52 | + HELM_EXTRA_FLAGS="--set=debugLogs=true,controller.sdkDebugLog=true" |
| 53 | +} |
| 54 | + |
| 55 | +param_set_infra() { |
| 56 | + GINKGO_FOCUS="\[param:(controllerReplicaCount|controllerPriorityClassName|controllerResources|controllerPodAnnotations|controllerPodLabels|controllerDeploymentAnnotations|controllerRevisionHistoryLimit|nodePriorityClassName|nodeResources|nodePodAnnotations|nodeDaemonSetAnnotations|nodeRevisionHistoryLimit|provisionerResources|attacherResources|snapshotterResources|resizerResources|nodeDriverRegistrarResources|livenessProbeResources|customLabels|controllerEnv|nodeEnv|controllerTopologySpreadConstraints|controllerSecurityContext|nodeSecurityContext|controllerContainerSecurityContext|controllerVolumes|controllerVolumeMounts|nodeVolumes|nodeVolumeMounts|controllerDnsConfig|nodeDnsConfig|controllerInitContainers|nodeInitContainers|imagePullPolicy|controllerUpdateStrategy|nodeUpdateStrategy|controllerTolerations|nodeTolerations|nameOverride|controllerAdditionalArgs|nodeAdditionalArgs|provisionerAdditionalArgs)\]" |
| 57 | + HELM_EXTRA_FLAGS="--set=controller.replicaCount=3,controller.priorityClassName=system-node-critical,controller.resources.requests.cpu=100m,controller.resources.limits.memory=256Mi,controller.podAnnotations.test-annotation=test-value,controller.podLabels.test-label=test-value,controller.deploymentAnnotations.deploy-annotation=deploy-value,controller.revisionHistoryLimit=5,node.priorityClassName=system-node-critical,node.resources.requests.cpu=50m,node.resources.limits.memory=128Mi,node.podAnnotations.node-annotation=node-value,node.daemonSetAnnotations.ds-annotation=ds-value,node.revisionHistoryLimit=3,sidecars.provisioner.resources.requests.cpu=20m,sidecars.attacher.resources.requests.cpu=15m,sidecars.snapshotter.resources.requests.cpu=15m,sidecars.resizer.resources.requests.cpu=15m,sidecars.nodeDriverRegistrar.resources.requests.cpu=10m,sidecars.livenessProbe.resources.requests.cpu=5m,customLabels.custom-label=custom-value,controller.env[0].name=TEST_ENV,controller.env[0].value=test-value,node.env[0].name=NODE_ENV,node.env[0].value=node-value,controller.topologySpreadConstraints[0].maxSkew=1,controller.topologySpreadConstraints[0].topologyKey=topology.kubernetes.io/zone,controller.topologySpreadConstraints[0].whenUnsatisfiable=ScheduleAnyway,controller.securityContext.runAsNonRoot=true,controller.containerSecurityContext.readOnlyRootFilesystem=true,controller.volumes[0].name=extra-volume,controller.volumes[0].configMap.name=kube-root-ca.crt,controller.volumeMounts[0].name=extra-volume,controller.volumeMounts[0].mountPath=/extra,node.volumes[0].name=node-extra-volume,node.volumes[0].configMap.name=kube-root-ca.crt,node.volumeMounts[0].name=node-extra-volume,node.volumeMounts[0].mountPath=/node-extra,controller.dnsConfig.nameservers[0]=8.8.8.8,node.dnsConfig.nameservers[0]=8.8.4.4,controller.initContainers[0].name=init-container,controller.initContainers[0].image=busybox,controller.initContainers[0].command[0]=echo,controller.initContainers[0].command[1]=init,node.initContainers[0].name=node-init-container,node.initContainers[0].image=busybox,node.initContainers[0].command[0]=echo,node.initContainers[0].command[1]=node-init,image.pullPolicy=Always,controller.updateStrategy.type=Recreate,controller.updateStrategy.rollingUpdate=null,node.updateStrategy.type=OnDelete,controller.tolerations[0].key=test-key,controller.tolerations[0].value=test-value,controller.tolerations[0].effect=NoSchedule,controller.tolerations[0].operator=Equal,nameOverride=custom-ebs-name,controller.additionalArgs[0]=--warn-on-invalid-tag,node.additionalArgs[0]=--logtostderr,sidecars.provisioner.additionalArgs[0]=--retry-interval-start=10s,node.tolerateAllTaints=false,node.tolerations[0].key=node-key,node.tolerations[0].value=node-value,node.tolerations[0].operator=Equal" |
| 58 | +} |
| 59 | + |
| 60 | +param_set_node-component-only() { |
| 61 | + GINKGO_FOCUS="\[param:nodeComponentOnly\]" |
| 62 | + HELM_EXTRA_FLAGS="--set=nodeComponentOnly=true" |
| 63 | +} |
| 64 | + |
| 65 | +param_set_legacy-compat() { |
| 66 | + GINKGO_FOCUS="\[param:(useOldCSIDriver|legacyXFS)\]" |
| 67 | + HELM_EXTRA_FLAGS="--set=useOldCSIDriver=true,node.legacyXFS=true" |
| 68 | +} |
| 69 | + |
| 70 | +param_set_selinux() { |
| 71 | + GINKGO_FOCUS="\[param:selinux\]" |
| 72 | + HELM_EXTRA_FLAGS="--set=node.selinux=true" |
| 73 | +} |
| 74 | + |
| 75 | +param_set_fips() { |
| 76 | + GINKGO_FOCUS="\[param:fips\]" |
| 77 | + HELM_EXTRA_FLAGS="--set=fips=true" |
| 78 | + FIPS_TEST=true make cluster/image |
| 79 | +} |
| 80 | + |
| 81 | +# Load a parameter set by name, exporting GINKGO_FOCUS and HELM_EXTRA_FLAGS |
| 82 | +load_param_set() { |
| 83 | + local name="$1" |
| 84 | + local func="param_set_${name}" |
| 85 | + if ! declare -f "$func" >/dev/null 2>&1; then |
| 86 | + echo "Unknown parameter set: ${name}" >&2 |
| 87 | + echo "Available sets: standard, other, debug, infra, legacy-compat, selinux, fips" >&2 |
| 88 | + exit 1 |
| 89 | + fi |
| 90 | + "$func" |
| 91 | + export GINKGO_FOCUS HELM_EXTRA_FLAGS |
| 92 | + export GINKGO_PARALLEL="${GINKGO_PARALLEL:-5}" |
| 93 | + export AWS_AVAILABILITY_ZONES="${AWS_AVAILABILITY_ZONES:-us-west-2a}" |
| 94 | + export TEST_PATH="${TEST_PATH:-./tests/e2e/...}" |
| 95 | + export JUNIT_REPORT="${REPORT_DIR:-/logs/artifacts}/junit-params-${name}.xml" |
| 96 | + # Export optional vars if set by the param set function |
| 97 | + if [[ -n "${EBS_INSTALL_SNAPSHOT+x}" ]]; then export EBS_INSTALL_SNAPSHOT; fi |
| 98 | + if [[ -n "${FIPS_TEST+x}" ]]; then export FIPS_TEST; fi |
| 99 | +} |
| 100 | + |
| 101 | +# Run a single parameter set |
| 102 | +run_param_set() { |
| 103 | + load_param_set "$1" |
| 104 | + echo "### Running parameter set: $1" |
| 105 | + ./hack/e2e/run.sh |
| 106 | +} |
| 107 | + |
| 108 | +# Merge per-set JUnit XMLs into a single file with duplicate skipped tests removed. |
| 109 | +# Each Ginkgo run reports ALL specs (most as skipped), so the same skipped test appears |
| 110 | +# in every per-set file. This merges all results into one file, keeping non-skipped results |
| 111 | +# (passed/failed) over skipped duplicates, and emitting each skipped test only once. |
| 112 | +merge_junit_results() { |
| 113 | + local report_dir="${REPORT_DIR:-/logs/artifacts}" |
| 114 | + local output="${report_dir}/junit-params.xml" |
| 115 | + |
| 116 | + python3 - "$report_dir" "$output" <<'PYEOF' |
| 117 | +import glob, sys, xml.etree.ElementTree as ET |
| 118 | +
|
| 119 | +report_dir, output = sys.argv[1], sys.argv[2] |
| 120 | +merged = {} |
| 121 | +time_total = 0.0 |
| 122 | +
|
| 123 | +def priority(tc): |
| 124 | + if tc.find("failure") is not None or tc.find("error") is not None: |
| 125 | + return 2 # failed/errored |
| 126 | + if tc.find("skipped") is not None: |
| 127 | + return 0 # skipped |
| 128 | + return 1 # passed |
| 129 | +
|
| 130 | +for path in sorted(glob.glob(f"{report_dir}/junit-params-*.xml")): |
| 131 | + tree = ET.parse(path) |
| 132 | + time_total += float(tree.getroot().get("time", "0")) |
| 133 | + for tc in tree.iter("testcase"): |
| 134 | + name = tc.get("name", "") |
| 135 | + if name not in merged or priority(tc) > priority(merged[name]): |
| 136 | + merged[name] = tc |
| 137 | +
|
| 138 | +tests = list(merged.values()) |
| 139 | +skipped = sum(1 for tc in tests if tc.find("skipped") is not None) |
| 140 | +failed = sum(1 for tc in tests if tc.find("failure") is not None) |
| 141 | +errored = sum(1 for tc in tests if tc.find("error") is not None) |
| 142 | +
|
| 143 | +root = ET.Element("testsuites", tests=str(len(tests)), disabled=str(skipped), |
| 144 | + errors=str(errored), failures=str(failed), time=str(time_total)) |
| 145 | +suite = ET.SubElement(root, "testsuite", name="AWS EBS CSI Driver Parameter Tests", |
| 146 | + tests=str(len(tests)), skipped=str(skipped), |
| 147 | + errors=str(errored), failures=str(failed), time=str(time_total)) |
| 148 | +for tc in tests: |
| 149 | + suite.append(tc) |
| 150 | +
|
| 151 | +ET.ElementTree(root).write(output, xml_declaration=True, encoding="UTF-8") |
| 152 | +PYEOF |
| 153 | + |
| 154 | + # Remove per-set files only if merge succeeded, so CI only sees the merged result |
| 155 | + if [[ $? -eq 0 && -f "$output" ]]; then |
| 156 | + rm -f "${report_dir}"/junit-params-*.xml |
| 157 | + echo "Merged JUnit results into ${output}" |
| 158 | + else |
| 159 | + echo "WARNING: JUnit merge failed, keeping per-set files" >&2 |
| 160 | + fi |
| 161 | +} |
| 162 | + |
| 163 | +# Run all standard parameter sets sequentially |
| 164 | +run_all_param_sets() { |
| 165 | + echo "Running all parameter sets sequentially..." |
| 166 | + local failed=0 |
| 167 | + for set in $PARAM_SETS_ALL; do |
| 168 | + run_param_set "$set" || failed=1 |
| 169 | + done |
| 170 | + merge_junit_results |
| 171 | + if [[ $failed -ne 0 ]]; then |
| 172 | + echo "One or more parameter sets failed!" |
| 173 | + return 1 |
| 174 | + fi |
| 175 | + echo "All parameter sets completed successfully!" |
| 176 | +} |
| 177 | + |
| 178 | +# Allow direct invocation: ./hack/e2e/param-sets.sh run <name> or ./hack/e2e/param-sets.sh run-all |
| 179 | +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then |
| 180 | + case "${1:-}" in |
| 181 | + run) |
| 182 | + [[ -z "${2:-}" ]] && { |
| 183 | + echo "Usage: $0 run <param-set-name>" >&2 |
| 184 | + exit 1 |
| 185 | + } |
| 186 | + run_param_set "$2" |
| 187 | + merge_junit_results |
| 188 | + ;; |
| 189 | + run-all) |
| 190 | + run_all_param_sets |
| 191 | + ;; |
| 192 | + *) |
| 193 | + echo "Usage: $0 {run <param-set-name>|run-all}" >&2 |
| 194 | + exit 1 |
| 195 | + ;; |
| 196 | + esac |
| 197 | +fi |
0 commit comments