From 90e7279447564bec438c9fec9159eeafa179a121 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Wed, 27 May 2026 20:11:09 +0300 Subject: [PATCH 1/4] Enable rhel98-brew-lrel-tuned image creation --- .../group3/rhel98-brew-lrel-tuned.toml | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-tuned.toml diff --git a/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-tuned.toml b/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-tuned.toml new file mode 100644 index 0000000000..5b1ebba293 --- /dev/null +++ b/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-tuned.toml @@ -0,0 +1,125 @@ +{{- if and (env.Getenv "BREW_LREL_RELEASE_VERSION" "") (env.Getenv "BREW_Y1_RELEASE_VERSION" "") -}} +{{- /* + + We wrap this template in a test so that the body of the output is + empty when there is no "current" version release. The output file + must end up completely empty, so we need to remove whitespace from + around the first and last template instructions. + +*/ -}} + +name = "rhel98-brew-lrel-tuned" +description = "A RHEL 9.8 image with already built and released RPMs like EC, RC, or Z-stream release: {{ .Env.BREW_LREL_RELEASE_VERSION }}" +version = "0.0.1" +modules = [] +groups = [] +distro = "rhel-98" + +# Parent specification directive recognized by test/bin/build_images.sh to be +# used with the '--parent' argument of 'osbuild-composer' +# parent = "rhel-9.6-microshift-brew-optionals-4.{{ .Env.PREVIOUS_MINOR_VERSION }}-zstream" + +{{ range (env.Getenv "MICROSHIFT_MANDATORY_RPMS" | strings.Split " ") }} +[[packages]] +name = "{{ . }}" +version = "{{ env.Getenv "BREW_LREL_RELEASE_VERSION" }}" +{{ end }} + +{{ range (env.Getenv "MICROSHIFT_OPTIONAL_RPMS" | strings.Split " ") }} +[[packages]] +name = "{{ . }}" +version = "{{ env.Getenv "BREW_LREL_RELEASE_VERSION" }}" +{{ end }} + +{{- if and (env.Getenv "UNAME_M" "") (eq "x86_64" .Env.UNAME_M) }} +{{ range (env.Getenv "MICROSHIFT_X86_64_RPMS" | strings.Split " ") }} +[[packages]] +name = "{{ . }}" +version = "{{ env.Getenv "BREW_LREL_RELEASE_VERSION" }}" +{{ end }} +{{- end }} + +[[packages]] +name = "microshift-test-agent" +version = "*" + +[[packages]] +name = "systemd-resolved" +version = "*" + +{{- if and (env.Getenv "UNAME_M" "") (eq "x86_64" .Env.UNAME_M) }} +# Kernel RT is only available for x86_64 +[customizations.kernel] +name = "kernel-rt" +{{- end }} + +[customizations.services] +enabled = ["microshift-test-agent", "microshift", "microshift-tuned"] + +[customizations.firewall] +ports = [ + "22:tcp", + "80:tcp", + "443:tcp", + "5353:udp", + "6443:tcp", + "8889:tcp", + "30000-32767:tcp", + "30000-32767:udp", +] + +[customizations.firewall.services] +enabled = ["mdns", "ssh", "http", "https"] + +[[customizations.firewall.zones]] +name = "trusted" +sources = ["10.42.0.0/16", "169.254.169.1", "fd01::/48"] + +[[customizations.files]] +path = "/etc/microshift/config.yaml" +data = """ +kubelet: + cpuManagerPolicy: static + cpuManagerPolicyOptions: + full-pcpus-only: "true" + cpuManagerReconcilePeriod: 5s + memoryManagerPolicy: Static + topologyManagerPolicy: single-numa-node + reservedSystemCPUs: 0-1 + reservedMemory: + - limits: + memory: 1100Mi + numaNode: 0 + kubeReserved: + memory: 500Mi + systemReserved: + memory: 500Mi + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s +""" + +[[customizations.files]] +path = "/etc/tuned/microshift-baseline-variables.conf" +data = """ +# Isolated cores should be complementary to kubelet's reserved CPUs. +# Isolated and reserved CPUs should contain all online CPUs. +# Core #3 is for testing offlining hence skipped. +isolated_cores=2,4-5 +hugepages_size=2M +hugepages=10 +additional_args=test1=on test2=true dummy +offline_cpu_set=3 +""" + +[[customizations.files]] +path = "/etc/microshift/tuned.yaml" +data = """ +profile: microshift-baseline +reboot_after_apply: True +""" + +{{- end -}} From 9463ea71cb442a1ba3927ae4666d3383b4dc16b8 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Wed, 27 May 2026 20:12:12 +0300 Subject: [PATCH 2/4] Stop microshift service before applying configuration changes --- .../rhel98-brew-lrel-tuned.toml.disabled | 125 ------------------ .../el102-lrel@multi-config-standard1.sh | 8 +- .../el102-lrel@multi-config-standard2.sh | 7 +- .../el98-lrel@multi-config-standard1.sh | 8 +- .../el98-lrel@multi-config-standard2.sh | 7 +- .../releases/el98-lrel@ginkgo-multi-config.sh | 40 +++--- 6 files changed, 40 insertions(+), 155 deletions(-) delete mode 100644 test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-tuned.toml.disabled diff --git a/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-tuned.toml.disabled b/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-tuned.toml.disabled deleted file mode 100644 index 5b1ebba293..0000000000 --- a/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-tuned.toml.disabled +++ /dev/null @@ -1,125 +0,0 @@ -{{- if and (env.Getenv "BREW_LREL_RELEASE_VERSION" "") (env.Getenv "BREW_Y1_RELEASE_VERSION" "") -}} -{{- /* - - We wrap this template in a test so that the body of the output is - empty when there is no "current" version release. The output file - must end up completely empty, so we need to remove whitespace from - around the first and last template instructions. - -*/ -}} - -name = "rhel98-brew-lrel-tuned" -description = "A RHEL 9.8 image with already built and released RPMs like EC, RC, or Z-stream release: {{ .Env.BREW_LREL_RELEASE_VERSION }}" -version = "0.0.1" -modules = [] -groups = [] -distro = "rhel-98" - -# Parent specification directive recognized by test/bin/build_images.sh to be -# used with the '--parent' argument of 'osbuild-composer' -# parent = "rhel-9.6-microshift-brew-optionals-4.{{ .Env.PREVIOUS_MINOR_VERSION }}-zstream" - -{{ range (env.Getenv "MICROSHIFT_MANDATORY_RPMS" | strings.Split " ") }} -[[packages]] -name = "{{ . }}" -version = "{{ env.Getenv "BREW_LREL_RELEASE_VERSION" }}" -{{ end }} - -{{ range (env.Getenv "MICROSHIFT_OPTIONAL_RPMS" | strings.Split " ") }} -[[packages]] -name = "{{ . }}" -version = "{{ env.Getenv "BREW_LREL_RELEASE_VERSION" }}" -{{ end }} - -{{- if and (env.Getenv "UNAME_M" "") (eq "x86_64" .Env.UNAME_M) }} -{{ range (env.Getenv "MICROSHIFT_X86_64_RPMS" | strings.Split " ") }} -[[packages]] -name = "{{ . }}" -version = "{{ env.Getenv "BREW_LREL_RELEASE_VERSION" }}" -{{ end }} -{{- end }} - -[[packages]] -name = "microshift-test-agent" -version = "*" - -[[packages]] -name = "systemd-resolved" -version = "*" - -{{- if and (env.Getenv "UNAME_M" "") (eq "x86_64" .Env.UNAME_M) }} -# Kernel RT is only available for x86_64 -[customizations.kernel] -name = "kernel-rt" -{{- end }} - -[customizations.services] -enabled = ["microshift-test-agent", "microshift", "microshift-tuned"] - -[customizations.firewall] -ports = [ - "22:tcp", - "80:tcp", - "443:tcp", - "5353:udp", - "6443:tcp", - "8889:tcp", - "30000-32767:tcp", - "30000-32767:udp", -] - -[customizations.firewall.services] -enabled = ["mdns", "ssh", "http", "https"] - -[[customizations.firewall.zones]] -name = "trusted" -sources = ["10.42.0.0/16", "169.254.169.1", "fd01::/48"] - -[[customizations.files]] -path = "/etc/microshift/config.yaml" -data = """ -kubelet: - cpuManagerPolicy: static - cpuManagerPolicyOptions: - full-pcpus-only: "true" - cpuManagerReconcilePeriod: 5s - memoryManagerPolicy: Static - topologyManagerPolicy: single-numa-node - reservedSystemCPUs: 0-1 - reservedMemory: - - limits: - memory: 1100Mi - numaNode: 0 - kubeReserved: - memory: 500Mi - systemReserved: - memory: 500Mi - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s -""" - -[[customizations.files]] -path = "/etc/tuned/microshift-baseline-variables.conf" -data = """ -# Isolated cores should be complementary to kubelet's reserved CPUs. -# Isolated and reserved CPUs should contain all online CPUs. -# Core #3 is for testing offlining hence skipped. -isolated_cores=2,4-5 -hugepages_size=2M -hugepages=10 -additional_args=test1=on test2=true dummy -offline_cpu_set=3 -""" - -[[customizations.files]] -path = "/etc/microshift/tuned.yaml" -data = """ -profile: microshift-baseline -reboot_after_apply: True -""" - -{{- end -}} diff --git a/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard1.sh b/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard1.sh index e0e9a1b94e..72a57bb057 100644 --- a/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard1.sh +++ b/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard1.sh @@ -59,6 +59,9 @@ scenario_run_tests() { sleep 5 done + # Stop MicroShift to before applying TLS configuration + run_command_on_vm host1 "sudo systemctl stop microshift" || true + # Apply TLSv1.3 configuration via drop-in config echo "INFO: Configuring TLSv1.3..." run_command_on_vm host1 "sudo mkdir -p /etc/microshift/config.d" @@ -68,9 +71,8 @@ apiServer: minVersion: VersionTLS13 EOF" - # Restart MicroShift to apply TLS configuration - echo "INFO: Restarting MicroShift to apply TLS configuration..." - run_command_on_vm host1 "sudo systemctl restart microshift" + # Start MicroShift to apply TLS configuration + run_command_on_vm host1 "sudo systemctl start microshift" # Wait for MicroShift to be ready wait_for_microshift_to_be_ready host1 diff --git a/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard2.sh b/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard2.sh index 1a9bb37023..7d30e564f4 100644 --- a/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard2.sh +++ b/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard2.sh @@ -59,6 +59,9 @@ scenario_run_tests() { sleep 5 done + # Stop MicroShift to before applying TLS configuration + run_command_on_vm host1 "sudo systemctl stop microshift" || true + # Apply TLSv1.3 configuration via drop-in config echo "INFO: Configuring TLSv1.3..." run_command_on_vm host1 "sudo mkdir -p /etc/microshift/config.d" @@ -68,8 +71,8 @@ apiServer: minVersion: VersionTLS13 EOF" - # Restart MicroShift to apply TLS configuration - run_command_on_vm host1 "sudo systemctl restart microshift" + # Start MicroShift to apply TLS configuration + run_command_on_vm host1 "sudo systemctl start microshift" # Wait for MicroShift to be ready wait_for_microshift_to_be_ready host1 diff --git a/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard1.sh b/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard1.sh index dd3304fb00..c1becfa6dd 100644 --- a/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard1.sh +++ b/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard1.sh @@ -59,6 +59,9 @@ scenario_run_tests() { sleep 5 done + # Stop MicroShift to before applying TLS configuration + run_command_on_vm host1 "sudo systemctl stop microshift" || true + # Apply TLSv1.3 configuration via drop-in config echo "INFO: Configuring TLSv1.3..." run_command_on_vm host1 "sudo mkdir -p /etc/microshift/config.d" @@ -68,9 +71,8 @@ apiServer: minVersion: VersionTLS13 EOF" - # Restart MicroShift to apply TLS configuration - echo "INFO: Restarting MicroShift to apply TLS configuration..." - run_command_on_vm host1 "sudo systemctl restart microshift" + # Start MicroShift to apply TLS configuration + run_command_on_vm host1 "sudo systemctl start microshift" # Wait for MicroShift to be ready wait_for_microshift_to_be_ready host1 diff --git a/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard2.sh b/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard2.sh index cd40ac76ba..e4cd098cb0 100644 --- a/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard2.sh +++ b/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard2.sh @@ -68,6 +68,9 @@ scenario_run_tests() { sleep 5 done + # Stop MicroShift to before applying TLS configuration + run_command_on_vm host1 "sudo systemctl stop microshift" || true + # Apply TLSv1.3 configuration via drop-in config echo "INFO: Configuring TLSv1.3..." run_command_on_vm host1 "sudo mkdir -p /etc/microshift/config.d" @@ -77,8 +80,8 @@ apiServer: minVersion: VersionTLS13 EOF" - # Restart MicroShift to apply TLS configuration - run_command_on_vm host1 "sudo systemctl restart microshift" + # Start MicroShift to apply TLS configuration + run_command_on_vm host1 "sudo systemctl start microshift" # Wait for MicroShift to be ready wait_for_microshift_to_be_ready host1 diff --git a/test/scenarios/releases/el98-lrel@ginkgo-multi-config.sh b/test/scenarios/releases/el98-lrel@ginkgo-multi-config.sh index 3571c9c943..89754c4259 100644 --- a/test/scenarios/releases/el98-lrel@ginkgo-multi-config.sh +++ b/test/scenarios/releases/el98-lrel@ginkgo-multi-config.sh @@ -17,8 +17,7 @@ VM_BRIDGE_IP="$(get_vm_bridge_ip "${VM_IPV6_NETWORK}")" # shellcheck disable=SC2034 # used elsewhere WEB_SERVER_URL="http://[${VM_BRIDGE_IP}]:${WEB_SERVER_PORT}" -# TODO: Consider using tuned image once it is enabled in the build system -start_image="rhel98-brew-lrel-optional" +start_image="rhel98-brew-lrel-tuned" scenario_create_vms() { exit_if_commit_not_found "${start_image}" @@ -48,22 +47,23 @@ scenario_remove_vms() { scenario_run_tests() { exit_if_commit_not_found "${start_image}" - # TODO: Re-enable once kernel-rt is available for RHEL 9.8 and the - # rhel98-brew-lrel-tuned image is used as start_image. - # # Wait for microshift-tuned to reboot the node - # local -r start_time=$(date +%s) - # while true; do - # boot_num=$(run_command_on_vm host1 "sudo journalctl --list-boots --quiet | wc -l" || true) - # boot_num="${boot_num%$'\r'*}" - # if [[ "${boot_num}" -ge 2 ]]; then - # break - # fi - # if [ $(( $(date +%s) - start_time )) -gt 60 ]; then - # echo "Timed out waiting for VM having 2 boots" - # exit 1 - # fi - # sleep 5 - # done + # Wait for microshift-tuned to reboot the node + local -r start_time=$(date +%s) + while true; do + boot_num=$(run_command_on_vm host1 "sudo journalctl --list-boots --quiet | wc -l" || true) + boot_num="${boot_num%$'\r'*}" + if [[ "${boot_num}" -ge 2 ]]; then + break + fi + if [ $(( $(date +%s) - start_time )) -gt 60 ]; then + echo "Timed out waiting for VM having 2 boots" + exit 1 + fi + sleep 5 + done + + # Stop MicroShift to before applying TLS configuration + run_command_on_vm host1 "sudo systemctl stop microshift" || true # Apply TLSv1.3 configuration via drop-in config echo "INFO: Configuring TLSv1.3..." @@ -74,8 +74,8 @@ apiServer: minVersion: VersionTLS13 EOF" - # Restart MicroShift to apply TLS configuration - run_command_on_vm host1 "sudo systemctl restart microshift" + # Start MicroShift to apply TLS configuration + run_command_on_vm host1 "sudo systemctl start microshift" # Wait for MicroShift to be ready wait_for_microshift_to_be_ready host1 From cb8a2ad8ffdbef02279bb2bf9c3368e730b4c014 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Wed, 27 May 2026 20:46:26 +0300 Subject: [PATCH 3/4] Fix remote ssh command invocation to avoid running trap in the function scope --- test/bin/scenario.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 678ee7d39b..bae8fce37f 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -111,7 +111,12 @@ run_command_on_vm() { # Necessary in devenv for entering input i.e. system registration, etc. term_opt="-t" fi - ssh "redhat@${ip}" -p "${ssh_port}" ${term_opt} "${command}" + + # Must return normally so local variables are cleaned up on command failure, + # otherwise stale values and readonly attributes break SOS collection + local rc=0 + ssh "redhat@${ip}" -p "${ssh_port}" ${term_opt} "${command}" || rc=$? + return "${rc}" } copy_file_to_vm() { @@ -126,7 +131,11 @@ copy_file_to_vm() { fi local -r ssh_port=$(get_vm_property "${vmname}" ssh_port) - scp -P "${ssh_port}" "${local_filename}" "redhat@${ip}:${remote_filename}" + # Must return normally so local variables are cleaned up on command failure, + # otherwise stale values and readonly attributes break SOS collection + local rc=0 + scp -P "${ssh_port}" "${local_filename}" "redhat@${ip}:${remote_filename}" || rc=$? + return "${rc}" } copy_file_from_vm() { @@ -141,7 +150,11 @@ copy_file_from_vm() { fi local -r ssh_port=$(get_vm_property "${vmname}" ssh_port) - scp -P "${ssh_port}" "redhat@${ip}:${remote_filename}" "${local_filename}" + # Must return normally so local variables are cleaned up on command failure, + # otherwise stale values and readonly attributes break SOS collection + local rc=0 + scp -P "${ssh_port}" "redhat@${ip}:${remote_filename}" "${local_filename}" || rc=$? + return "${rc}" } sos_report() { From 9df75a085552936a29a0d8bb32e83c9bf050b6d8 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Wed, 27 May 2026 23:33:04 +0300 Subject: [PATCH 4/4] Fix comment typo --- .../el10/releases/el102-lrel@multi-config-standard1.sh | 2 +- .../el10/releases/el102-lrel@multi-config-standard2.sh | 2 +- .../el9/releases/el98-lrel@multi-config-standard1.sh | 2 +- .../el9/releases/el98-lrel@multi-config-standard2.sh | 2 +- test/scenarios/releases/el98-lrel@ginkgo-multi-config.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard1.sh b/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard1.sh index 72a57bb057..dcbbcf58dc 100644 --- a/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard1.sh +++ b/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard1.sh @@ -59,7 +59,7 @@ scenario_run_tests() { sleep 5 done - # Stop MicroShift to before applying TLS configuration + # Stop MicroShift before applying TLS configuration run_command_on_vm host1 "sudo systemctl stop microshift" || true # Apply TLSv1.3 configuration via drop-in config diff --git a/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard2.sh b/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard2.sh index 7d30e564f4..a33566dfae 100644 --- a/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard2.sh +++ b/test/scenarios-bootc/el10/releases/el102-lrel@multi-config-standard2.sh @@ -59,7 +59,7 @@ scenario_run_tests() { sleep 5 done - # Stop MicroShift to before applying TLS configuration + # Stop MicroShift before applying TLS configuration run_command_on_vm host1 "sudo systemctl stop microshift" || true # Apply TLSv1.3 configuration via drop-in config diff --git a/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard1.sh b/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard1.sh index c1becfa6dd..df93286a50 100644 --- a/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard1.sh +++ b/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard1.sh @@ -59,7 +59,7 @@ scenario_run_tests() { sleep 5 done - # Stop MicroShift to before applying TLS configuration + # Stop MicroShift before applying TLS configuration run_command_on_vm host1 "sudo systemctl stop microshift" || true # Apply TLSv1.3 configuration via drop-in config diff --git a/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard2.sh b/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard2.sh index e4cd098cb0..18e0db5637 100644 --- a/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard2.sh +++ b/test/scenarios-bootc/el9/releases/el98-lrel@multi-config-standard2.sh @@ -68,7 +68,7 @@ scenario_run_tests() { sleep 5 done - # Stop MicroShift to before applying TLS configuration + # Stop MicroShift before applying TLS configuration run_command_on_vm host1 "sudo systemctl stop microshift" || true # Apply TLSv1.3 configuration via drop-in config diff --git a/test/scenarios/releases/el98-lrel@ginkgo-multi-config.sh b/test/scenarios/releases/el98-lrel@ginkgo-multi-config.sh index 89754c4259..dcedffbd71 100644 --- a/test/scenarios/releases/el98-lrel@ginkgo-multi-config.sh +++ b/test/scenarios/releases/el98-lrel@ginkgo-multi-config.sh @@ -62,7 +62,7 @@ scenario_run_tests() { sleep 5 done - # Stop MicroShift to before applying TLS configuration + # Stop MicroShift before applying TLS configuration run_command_on_vm host1 "sudo systemctl stop microshift" || true # Apply TLSv1.3 configuration via drop-in config