From 2a6e7ad74ac6b0bcb694ecfa928a2199e691c12c Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Wed, 13 May 2026 10:32:06 +0200 Subject: [PATCH 1/5] USHIFT-6902: Add PCP metrics collection to test scenarios Bake pcp-zeroconf into all test VM images (ostree blueprints across all layers and bootc Containerfiles) so Performance Co-Pilot is available on every test VM. At test time, scenario.sh starts pmcd/pmlogger before tests and collects the PCP archives as artifacts alongside SOS reports. Controlled via SKIP_PCP environment variable (defaults to false). Co-Authored-By: Claude Opus 4.6 --- test/bin/scenario.sh | 72 ++++++++++++++++++- .../group1/rhel102-test-agent.containerfile | 2 +- .../group1/rhel96-test-agent.containerfile | 2 +- .../group1/rhel98-test-agent.containerfile | 2 +- .../group1/cos10-test-agent.containerfile | 2 +- .../group1/cos9-test-agent.containerfile | 2 +- .../layer1-base/group1/rhel96.toml | 4 ++ .../layer1-base/group1/rhel98.toml | 4 ++ .../group1/rhel98-source-base.toml | 4 ++ .../group1/rhel98-source-fake-next-minor.toml | 4 ++ .../group1/rhel98-source-with-optionals.toml | 4 ++ .../group1/rhel98-source.toml | 4 ++ .../group1/rhel98-source-isolated.toml | 4 ++ .../group1/rhel96-brew-y2-with-optionals.toml | 4 ++ .../group2/rhel96-brew-y1-with-optionals.toml | 4 ++ .../group3/rhel98-brew-lrel-optional.toml | 4 ++ .../rhel98-brew-nightly-with-optionals.toml | 4 ++ 17 files changed, 120 insertions(+), 6 deletions(-) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 2053b7b276..93e2c28c83 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -22,6 +22,7 @@ PULL_SECRET_CONTENT="$(jq -c . "${PULL_SECRET}")" VM_BOOT_TIMEOUT=1200 # Overall total boot times are around 15m VM_GREENBOOT_TIMEOUT=1800 # Greenboot readiness may take up to 15-30m depending on the load SKIP_SOS=${SKIP_SOS:-false} # may be overridden in global settings file +SKIP_PCP=${SKIP_PCP:-false} # may be overridden in global settings file SKIP_GREENBOOT=${SKIP_GREENBOOT:-false} # may be overridden in scenario file GREENBOOT_TIMEOUT=${GREENBOOT_TIMEOUT:-600} # may be overridden in scenario file # Container image signature verification should be disabled by default in the @@ -292,6 +293,73 @@ sos_report_for_vm_offline() { "--filename" "*.log" } +start_pcp_on_all_vms() { + if "${SKIP_PCP}"; then + echo "Skipping PCP collection" + return 0 + fi + + for vmdir in "${SCENARIO_INFO_DIR}"/"${SCENARIO}"/vms/*; do + if [ ! -d "${vmdir}" ]; then + continue + fi + + local vmname + vmname=$(basename "${vmdir}") + local ip + ip=$(cat "$(vm_property_filename "${vmname}" "ip")" 2>/dev/null) || true + + if [ -z "${ip}" ]; then + continue + fi + + echo "Starting PCP collection on ${vmname}" + run_command_on_vm "${vmname}" \ + "sudo systemctl restart pmcd && \ + sudo systemctl restart pmlogger" \ + || echo "WARNING: Failed to start PCP on ${vmname}" + done +} + +collect_pcp_reports() { + if "${SKIP_PCP}"; then + echo "Skipping PCP collection" + return 0 + fi + + echo "Collecting PCP reports" + for vmdir in "${SCENARIO_INFO_DIR}"/"${SCENARIO}"/vms/*; do + if [ ! -d "${vmdir}" ]; then + continue + fi + + local vmname + vmname=$(basename "${vmdir}") + local ip + ip=$(cat "$(vm_property_filename "${vmname}" "ip")" 2>/dev/null) || true + + if [ -z "${ip}" ]; then + continue + fi + + echo "Collecting PCP data from ${vmname}" + run_command_on_vm "${vmname}" "sudo systemctl stop pmlogger" || true + + if ! run_command_on_vm "${vmname}" "test -d /var/log/pcp/pmlogger" ; then + echo "WARNING: No PCP data directory on ${vmname}, skipping collection" + continue + fi + + run_command_on_vm "${vmname}" \ + "sudo tar czf /tmp/pcp-archives.tar.gz -C /var/log/pcp/pmlogger ." || true + + mkdir -p "${vmdir}/pcp" + copy_file_from_vm "${vmname}" "/tmp/pcp-archives.tar.gz" "${vmdir}/pcp/" || { + echo "WARNING: Failed to collect PCP data from ${vmname}" + } + done +} + get_lrel_release_image_url() { local -r brew_lrel_release_version="$1" local image_url="" @@ -1584,13 +1652,14 @@ action_run() { fi record_junit "run" "load_scenario_script" "OK" - # Set the exit handler to attempt the sos report collection and error logging + # Set the exit handler to attempt PCP and SOS report collection and error logging # - Preserve the original exit code # - Log junit message on failure # - Override the exit code if sos report collection fails # shellcheck disable=SC2154 trap 'rc=$? ; \ [ "${rc}" -ne 0 ] && record_junit "run" "scenario_run_tests" "FAILED" ; \ + collect_pcp_reports || true ; \ sos_report true || rc=1 ; \ close_junit ; exit "${rc}"' EXIT @@ -1600,6 +1669,7 @@ action_run() { else RUN_HOST_OVERRIDE="$1" fi + start_pcp_on_all_vms scenario_run_tests record_junit "run" "scenario_run_tests" "OK" } diff --git a/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile b/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile index d4bf2dbf96..7bcf5940ae 100644 --- a/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile +++ b/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile @@ -22,7 +22,7 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup -RUN dnf install -y microshift-test-agent && \ +RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ systemctl enable microshift-test-agent && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ diff --git a/test/image-blueprints-bootc/el9/layer1-base/group1/rhel96-test-agent.containerfile b/test/image-blueprints-bootc/el9/layer1-base/group1/rhel96-test-agent.containerfile index d08f10fb5d..a5ed225a74 100644 --- a/test/image-blueprints-bootc/el9/layer1-base/group1/rhel96-test-agent.containerfile +++ b/test/image-blueprints-bootc/el9/layer1-base/group1/rhel96-test-agent.containerfile @@ -15,7 +15,7 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup -RUN dnf install -y microshift-test-agent && \ +RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ systemctl enable microshift-test-agent && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ diff --git a/test/image-blueprints-bootc/el9/layer1-base/group1/rhel98-test-agent.containerfile b/test/image-blueprints-bootc/el9/layer1-base/group1/rhel98-test-agent.containerfile index 7ac553d111..1f5f1f6552 100644 --- a/test/image-blueprints-bootc/el9/layer1-base/group1/rhel98-test-agent.containerfile +++ b/test/image-blueprints-bootc/el9/layer1-base/group1/rhel98-test-agent.containerfile @@ -22,7 +22,7 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup -RUN dnf install -y microshift-test-agent && \ +RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ systemctl enable microshift-test-agent && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ diff --git a/test/image-blueprints-bootc/upstream/group1/cos10-test-agent.containerfile b/test/image-blueprints-bootc/upstream/group1/cos10-test-agent.containerfile index 982ba1bf45..e04070ceb5 100644 --- a/test/image-blueprints-bootc/upstream/group1/cos10-test-agent.containerfile +++ b/test/image-blueprints-bootc/upstream/group1/cos10-test-agent.containerfile @@ -15,7 +15,7 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup -RUN dnf install -y microshift-test-agent && \ +RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ systemctl enable microshift-test-agent && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ diff --git a/test/image-blueprints-bootc/upstream/group1/cos9-test-agent.containerfile b/test/image-blueprints-bootc/upstream/group1/cos9-test-agent.containerfile index 58efa66116..d7b3065674 100644 --- a/test/image-blueprints-bootc/upstream/group1/cos9-test-agent.containerfile +++ b/test/image-blueprints-bootc/upstream/group1/cos9-test-agent.containerfile @@ -15,7 +15,7 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup -RUN dnf install -y microshift-test-agent && \ +RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ systemctl enable microshift-test-agent && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ diff --git a/test/image-blueprints/layer1-base/group1/rhel96.toml b/test/image-blueprints/layer1-base/group1/rhel96.toml index 6334d3a567..591cf8e669 100644 --- a/test/image-blueprints/layer1-base/group1/rhel96.toml +++ b/test/image-blueprints/layer1-base/group1/rhel96.toml @@ -13,6 +13,10 @@ version = "*" name = "iproute-tc" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [customizations.services] enabled = ["microshift-test-agent"] diff --git a/test/image-blueprints/layer1-base/group1/rhel98.toml b/test/image-blueprints/layer1-base/group1/rhel98.toml index 7f9dc37a14..682f34338a 100644 --- a/test/image-blueprints/layer1-base/group1/rhel98.toml +++ b/test/image-blueprints/layer1-base/group1/rhel98.toml @@ -13,6 +13,10 @@ version = "*" name = "iproute-tc" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [customizations.services] enabled = ["microshift-test-agent"] diff --git a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-base.toml b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-base.toml index 89e93d4b37..8a591a9e8a 100644 --- a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-base.toml +++ b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-base.toml @@ -25,6 +25,10 @@ version = "{{ env.Getenv "SOURCE_VERSION_BASE" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [customizations.services] enabled = ["microshift", "microshift-test-agent"] diff --git a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-fake-next-minor.toml b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-fake-next-minor.toml index d16fb10cf9..2f06f59dc7 100644 --- a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-fake-next-minor.toml +++ b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-fake-next-minor.toml @@ -35,6 +35,10 @@ version = "{{ .Env.FAKE_NEXT_MAJOR_VERSION }}.{{ .Env.FAKE_NEXT_MINOR_VERSION }} name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [customizations.services] enabled = ["microshift", "microshift-test-agent"] diff --git a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-with-optionals.toml b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-with-optionals.toml index 384606bb8b..6259f71494 100644 --- a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-with-optionals.toml +++ b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-with-optionals.toml @@ -39,6 +39,10 @@ version = "{{ env.Getenv "SOURCE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [[packages]] name = "systemd-resolved" version = "*" diff --git a/test/image-blueprints/layer2-presubmit/group1/rhel98-source.toml b/test/image-blueprints/layer2-presubmit/group1/rhel98-source.toml index 61d2e88732..f75b1ebc0f 100644 --- a/test/image-blueprints/layer2-presubmit/group1/rhel98-source.toml +++ b/test/image-blueprints/layer2-presubmit/group1/rhel98-source.toml @@ -25,6 +25,10 @@ version = "{{ env.Getenv "SOURCE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [[packages]] name = "systemd-resolved" version = "*" diff --git a/test/image-blueprints/layer3-periodic/group1/rhel98-source-isolated.toml b/test/image-blueprints/layer3-periodic/group1/rhel98-source-isolated.toml index 42e1729c05..8a605c0f93 100644 --- a/test/image-blueprints/layer3-periodic/group1/rhel98-source-isolated.toml +++ b/test/image-blueprints/layer3-periodic/group1/rhel98-source-isolated.toml @@ -15,6 +15,10 @@ version = "{{ env.Getenv "SOURCE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [[packages]] name = "qemu-guest-agent" version = "*" diff --git a/test/image-blueprints/layer4-release/group1/rhel96-brew-y2-with-optionals.toml b/test/image-blueprints/layer4-release/group1/rhel96-brew-y2-with-optionals.toml index 4f762059c7..c5f3dfae06 100644 --- a/test/image-blueprints/layer4-release/group1/rhel96-brew-y2-with-optionals.toml +++ b/test/image-blueprints/layer4-release/group1/rhel96-brew-y2-with-optionals.toml @@ -39,6 +39,10 @@ version = "{{ env.Getenv "BREW_Y2_RELEASE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [customizations.services] enabled = ["microshift", "microshift-test-agent"] diff --git a/test/image-blueprints/layer4-release/group2/rhel96-brew-y1-with-optionals.toml b/test/image-blueprints/layer4-release/group2/rhel96-brew-y1-with-optionals.toml index 22134c79ab..5d1c810586 100644 --- a/test/image-blueprints/layer4-release/group2/rhel96-brew-y1-with-optionals.toml +++ b/test/image-blueprints/layer4-release/group2/rhel96-brew-y1-with-optionals.toml @@ -43,6 +43,10 @@ version = "{{ env.Getenv "BREW_Y1_RELEASE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [customizations.services] enabled = ["microshift", "microshift-test-agent"] diff --git a/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-optional.toml b/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-optional.toml index c7353490e7..df3f877442 100644 --- a/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-optional.toml +++ b/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-optional.toml @@ -43,6 +43,10 @@ version = "{{ env.Getenv "BREW_LREL_RELEASE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [[packages]] name = "systemd-resolved" version = "*" diff --git a/test/image-blueprints/layer4-release/group3/rhel98-brew-nightly-with-optionals.toml b/test/image-blueprints/layer4-release/group3/rhel98-brew-nightly-with-optionals.toml index c32156e593..6ac32af03d 100644 --- a/test/image-blueprints/layer4-release/group3/rhel98-brew-nightly-with-optionals.toml +++ b/test/image-blueprints/layer4-release/group3/rhel98-brew-nightly-with-optionals.toml @@ -43,6 +43,10 @@ version = "{{ env.Getenv "BREW_NIGHTLY_RELEASE_VERSION" }}" name = "microshift-test-agent" version = "*" +[[packages]] +name = "pcp-zeroconf" +version = "*" + [[packages]] name = "systemd-resolved" version = "*" From 3d0afe01a1db58463d8c04afc8c5ecd315688ada Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Wed, 13 May 2026 15:08:47 +0200 Subject: [PATCH 2/5] USHIFT-6979: Enable PCP services at boot in ostree blueprints Add pmcd and pmlogger to the enabled services list in all ostree image blueprints that include pcp-zeroconf. This ensures PCP starts logging from first boot, capturing metrics during the boot and greenboot phases rather than only during test execution. --- test/image-blueprints/layer1-base/group1/rhel96.toml | 2 +- test/image-blueprints/layer1-base/group1/rhel98.toml | 2 +- .../layer2-presubmit/group1/rhel98-source-base.toml | 2 +- .../layer2-presubmit/group1/rhel98-source-fake-next-minor.toml | 2 +- .../layer2-presubmit/group1/rhel98-source-with-optionals.toml | 2 +- .../image-blueprints/layer2-presubmit/group1/rhel98-source.toml | 2 +- .../layer3-periodic/group1/rhel98-source-isolated.toml | 2 +- .../layer4-release/group1/rhel96-brew-y2-with-optionals.toml | 2 +- .../layer4-release/group2/rhel96-brew-y1-with-optionals.toml | 2 +- .../layer4-release/group3/rhel98-brew-lrel-optional.toml | 2 +- .../group3/rhel98-brew-nightly-with-optionals.toml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/image-blueprints/layer1-base/group1/rhel96.toml b/test/image-blueprints/layer1-base/group1/rhel96.toml index 591cf8e669..0b2dc21535 100644 --- a/test/image-blueprints/layer1-base/group1/rhel96.toml +++ b/test/image-blueprints/layer1-base/group1/rhel96.toml @@ -18,7 +18,7 @@ name = "pcp-zeroconf" version = "*" [customizations.services] -enabled = ["microshift-test-agent"] +enabled = ["microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp"] diff --git a/test/image-blueprints/layer1-base/group1/rhel98.toml b/test/image-blueprints/layer1-base/group1/rhel98.toml index 682f34338a..0f004f3828 100644 --- a/test/image-blueprints/layer1-base/group1/rhel98.toml +++ b/test/image-blueprints/layer1-base/group1/rhel98.toml @@ -18,7 +18,7 @@ name = "pcp-zeroconf" version = "*" [customizations.services] -enabled = ["microshift-test-agent"] +enabled = ["microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp"] diff --git a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-base.toml b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-base.toml index 8a591a9e8a..2f2b9d8056 100644 --- a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-base.toml +++ b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-base.toml @@ -30,7 +30,7 @@ name = "pcp-zeroconf" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"] diff --git a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-fake-next-minor.toml b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-fake-next-minor.toml index 2f06f59dc7..4ced4d29a5 100644 --- a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-fake-next-minor.toml +++ b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-fake-next-minor.toml @@ -40,7 +40,7 @@ name = "pcp-zeroconf" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"] diff --git a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-with-optionals.toml b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-with-optionals.toml index 6259f71494..64f6097657 100644 --- a/test/image-blueprints/layer2-presubmit/group1/rhel98-source-with-optionals.toml +++ b/test/image-blueprints/layer2-presubmit/group1/rhel98-source-with-optionals.toml @@ -48,7 +48,7 @@ name = "systemd-resolved" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"] diff --git a/test/image-blueprints/layer2-presubmit/group1/rhel98-source.toml b/test/image-blueprints/layer2-presubmit/group1/rhel98-source.toml index f75b1ebc0f..60bb03c2b7 100644 --- a/test/image-blueprints/layer2-presubmit/group1/rhel98-source.toml +++ b/test/image-blueprints/layer2-presubmit/group1/rhel98-source.toml @@ -34,7 +34,7 @@ name = "systemd-resolved" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"] diff --git a/test/image-blueprints/layer3-periodic/group1/rhel98-source-isolated.toml b/test/image-blueprints/layer3-periodic/group1/rhel98-source-isolated.toml index 8a605c0f93..51280fd0c4 100644 --- a/test/image-blueprints/layer3-periodic/group1/rhel98-source-isolated.toml +++ b/test/image-blueprints/layer3-periodic/group1/rhel98-source-isolated.toml @@ -38,7 +38,7 @@ name = "skopeo" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent", "qemu-guest-agent"] +enabled = ["microshift", "microshift-test-agent", "qemu-guest-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"] diff --git a/test/image-blueprints/layer4-release/group1/rhel96-brew-y2-with-optionals.toml b/test/image-blueprints/layer4-release/group1/rhel96-brew-y2-with-optionals.toml index c5f3dfae06..434b6f4f95 100644 --- a/test/image-blueprints/layer4-release/group1/rhel96-brew-y2-with-optionals.toml +++ b/test/image-blueprints/layer4-release/group1/rhel96-brew-y2-with-optionals.toml @@ -44,7 +44,7 @@ name = "pcp-zeroconf" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = [ diff --git a/test/image-blueprints/layer4-release/group2/rhel96-brew-y1-with-optionals.toml b/test/image-blueprints/layer4-release/group2/rhel96-brew-y1-with-optionals.toml index 5d1c810586..2bc677ee39 100644 --- a/test/image-blueprints/layer4-release/group2/rhel96-brew-y1-with-optionals.toml +++ b/test/image-blueprints/layer4-release/group2/rhel96-brew-y1-with-optionals.toml @@ -48,7 +48,7 @@ name = "pcp-zeroconf" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = [ diff --git a/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-optional.toml b/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-optional.toml index df3f877442..d35ca71203 100644 --- a/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-optional.toml +++ b/test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-optional.toml @@ -52,7 +52,7 @@ name = "systemd-resolved" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = [ diff --git a/test/image-blueprints/layer4-release/group3/rhel98-brew-nightly-with-optionals.toml b/test/image-blueprints/layer4-release/group3/rhel98-brew-nightly-with-optionals.toml index 6ac32af03d..b7d2f8232e 100644 --- a/test/image-blueprints/layer4-release/group3/rhel98-brew-nightly-with-optionals.toml +++ b/test/image-blueprints/layer4-release/group3/rhel98-brew-nightly-with-optionals.toml @@ -52,7 +52,7 @@ name = "systemd-resolved" version = "*" [customizations.services] -enabled = ["microshift", "microshift-test-agent"] +enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"] [customizations.firewall] ports = [ From 26ab45b7d0b806b7da636bd6c644c9cf00091501 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Wed, 13 May 2026 15:10:09 +0200 Subject: [PATCH 3/5] USHIFT-6979: Enable PCP services at boot in bootc containerfiles Add pmcd and pmlogger to the systemctl enable command in all bootc test-agent containerfiles that install pcp-zeroconf. This ensures PCP starts logging from first boot in bootc-based test VMs. --- .../el10/layer1-base/group1/rhel102-test-agent.containerfile | 2 +- .../el9/layer1-base/group1/rhel96-test-agent.containerfile | 2 +- .../el9/layer1-base/group1/rhel98-test-agent.containerfile | 2 +- .../upstream/group1/cos10-test-agent.containerfile | 2 +- .../upstream/group1/cos9-test-agent.containerfile | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile b/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile index 7bcf5940ae..9441297f95 100644 --- a/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile +++ b/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile @@ -23,7 +23,7 @@ COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ - systemctl enable microshift-test-agent && \ + systemctl enable microshift-test-agent pmcd pmlogger && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ dnf clean all diff --git a/test/image-blueprints-bootc/el9/layer1-base/group1/rhel96-test-agent.containerfile b/test/image-blueprints-bootc/el9/layer1-base/group1/rhel96-test-agent.containerfile index a5ed225a74..a872170aa7 100644 --- a/test/image-blueprints-bootc/el9/layer1-base/group1/rhel96-test-agent.containerfile +++ b/test/image-blueprints-bootc/el9/layer1-base/group1/rhel96-test-agent.containerfile @@ -16,7 +16,7 @@ COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ - systemctl enable microshift-test-agent && \ + systemctl enable microshift-test-agent pmcd pmlogger && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ dnf clean all diff --git a/test/image-blueprints-bootc/el9/layer1-base/group1/rhel98-test-agent.containerfile b/test/image-blueprints-bootc/el9/layer1-base/group1/rhel98-test-agent.containerfile index 1f5f1f6552..6d156f2fdb 100644 --- a/test/image-blueprints-bootc/el9/layer1-base/group1/rhel98-test-agent.containerfile +++ b/test/image-blueprints-bootc/el9/layer1-base/group1/rhel98-test-agent.containerfile @@ -23,7 +23,7 @@ COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ - systemctl enable microshift-test-agent && \ + systemctl enable microshift-test-agent pmcd pmlogger && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ dnf clean all diff --git a/test/image-blueprints-bootc/upstream/group1/cos10-test-agent.containerfile b/test/image-blueprints-bootc/upstream/group1/cos10-test-agent.containerfile index e04070ceb5..166d607d24 100644 --- a/test/image-blueprints-bootc/upstream/group1/cos10-test-agent.containerfile +++ b/test/image-blueprints-bootc/upstream/group1/cos10-test-agent.containerfile @@ -16,7 +16,7 @@ COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ - systemctl enable microshift-test-agent && \ + systemctl enable microshift-test-agent pmcd pmlogger && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ dnf clean all diff --git a/test/image-blueprints-bootc/upstream/group1/cos9-test-agent.containerfile b/test/image-blueprints-bootc/upstream/group1/cos9-test-agent.containerfile index d7b3065674..55224583d0 100644 --- a/test/image-blueprints-bootc/upstream/group1/cos9-test-agent.containerfile +++ b/test/image-blueprints-bootc/upstream/group1/cos9-test-agent.containerfile @@ -16,7 +16,7 @@ COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/ # Install the test agent and cleanup RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ - systemctl enable microshift-test-agent && \ + systemctl enable microshift-test-agent pmcd pmlogger && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ dnf clean all From 57e9358e37e22a936dc835e6f4a264192968aed6 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Wed, 13 May 2026 15:11:19 +0200 Subject: [PATCH 4/5] USHIFT-6979: Add offline PCP collection and remove manual start Since PCP services are now enabled at the image level, remove the start_pcp_on_all_vms function that manually restarted pmcd/pmlogger before tests. Refactor collect_pcp_reports to dispatch between online and offline collection, mirroring the sos_report pattern. Offline VMs (no network) are now collected via invoke_qemu_script. Also add PCP collection to the action_create EXIT trap so archives are captured even when the boot phase fails. --- test/bin/scenario.sh | 84 +++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 36 deletions(-) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 93e2c28c83..c69d98db47 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -293,32 +293,54 @@ sos_report_for_vm_offline() { "--filename" "*.log" } -start_pcp_on_all_vms() { - if "${SKIP_PCP}"; then - echo "Skipping PCP collection" +collect_pcp_reports_for_vm() { + local -r vmdir="${1}" + local -r vmname="${2}" + + echo "Collecting PCP data from ${vmname}" + run_command_on_vm "${vmname}" "sudo systemctl stop pmlogger" || true + + if ! run_command_on_vm "${vmname}" "test -d /var/log/pcp/pmlogger" ; then + echo "WARNING: No PCP data directory on ${vmname}, skipping collection" return 0 fi - for vmdir in "${SCENARIO_INFO_DIR}"/"${SCENARIO}"/vms/*; do - if [ ! -d "${vmdir}" ]; then - continue - fi + run_command_on_vm "${vmname}" \ + "sudo tar czf /tmp/pcp-archives.tar.gz -C /var/log/pcp/pmlogger ." || true - local vmname - vmname=$(basename "${vmdir}") - local ip - ip=$(cat "$(vm_property_filename "${vmname}" "ip")" 2>/dev/null) || true + mkdir -p "${vmdir}/pcp" + copy_file_from_vm "${vmname}" "/tmp/pcp-archives.tar.gz" "${vmdir}/pcp/" || { + echo "WARNING: Failed to collect PCP data from ${vmname}" + } +} - if [ -z "${ip}" ]; then - continue - fi +collect_pcp_reports_for_vm_offline() { + local -r vmdir="${1}" + local -r vmname="${2}" + local -r full_vmname="$(full_vm_name "${vmname}")" - echo "Starting PCP collection on ${vmname}" - run_command_on_vm "${vmname}" \ - "sudo systemctl restart pmcd && \ - sudo systemctl restart pmlogger" \ - || echo "WARNING: Failed to start PCP on ${vmname}" - done + echo "Collecting PCP data offline from ${vmname}" + + "${ROOTDIR}/scripts/fetch_tools.sh" "robotframework" + + invoke_qemu_script "wait" \ + "--vm" "${full_vmname}" + + invoke_qemu_script "bash" \ + "--vm" "${full_vmname}" \ + "--args" "sudo systemctl stop pmlogger" + + invoke_qemu_script "bash" \ + "--vm" "${full_vmname}" \ + "--args" "sudo tar czf /tmp/pcp-archives.tar.gz -C /var/log/pcp/pmlogger ." + + mkdir -p "${vmdir}/pcp" + + invoke_qemu_script "download" \ + "--vm" "${full_vmname}" \ + "--src_dir" "/tmp/" \ + "--dst_dir" "${vmdir}/pcp/" \ + "--filename" "pcp-archives.tar.gz" } collect_pcp_reports() { @@ -338,23 +360,13 @@ collect_pcp_reports() { local ip ip=$(cat "$(vm_property_filename "${vmname}" "ip")" 2>/dev/null) || true + local pcp_func="collect_pcp_reports_for_vm" if [ -z "${ip}" ]; then - continue + echo "Collecting PCP reports offline" + pcp_func="collect_pcp_reports_for_vm_offline" fi - echo "Collecting PCP data from ${vmname}" - run_command_on_vm "${vmname}" "sudo systemctl stop pmlogger" || true - - if ! run_command_on_vm "${vmname}" "test -d /var/log/pcp/pmlogger" ; then - echo "WARNING: No PCP data directory on ${vmname}, skipping collection" - continue - fi - - run_command_on_vm "${vmname}" \ - "sudo tar czf /tmp/pcp-archives.tar.gz -C /var/log/pcp/pmlogger ." || true - - mkdir -p "${vmdir}/pcp" - copy_file_from_vm "${vmname}" "/tmp/pcp-archives.tar.gz" "${vmdir}/pcp/" || { + "${pcp_func}" "${vmdir}" "${vmname}" || { echo "WARNING: Failed to collect PCP data from ${vmname}" } done @@ -1594,13 +1606,14 @@ action_create() { fi record_junit "setup" "load_scenario_script" "OK" - # Set the exit handler to attempt the sos report collection and error logging + # Set the exit handler to attempt PCP and SOS report collection and error logging # - Preserve the original exit code # - Log junit message on failure # - Override the exit code if sos report collection fails # shellcheck disable=SC2154 trap 'rc=$? ; \ [ "${rc}" -ne 0 ] && record_junit "setup" "scenario_create_vms" "FAILED" ; \ + collect_pcp_reports || true ; \ sos_report true || rc=1 ; \ close_junit ; exit "${rc}"' EXIT @@ -1669,7 +1682,6 @@ action_run() { else RUN_HOST_OVERRIDE="$1" fi - start_pcp_on_all_vms scenario_run_tests record_junit "run" "scenario_run_tests" "OK" } From 4e8402cd10ad1f64831a380071274c6e2dfad880 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Thu, 14 May 2026 08:24:39 +0200 Subject: [PATCH 5/5] USHIFT-6979: Nits --- test/bin/scenario.sh | 4 ++-- test/scenario_settings.sh.example | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index c69d98db47..678ee7d39b 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -298,7 +298,7 @@ collect_pcp_reports_for_vm() { local -r vmname="${2}" echo "Collecting PCP data from ${vmname}" - run_command_on_vm "${vmname}" "sudo systemctl stop pmlogger" || true + run_command_on_vm "${vmname}" "sudo systemctl stop --now pmlogger" || true if ! run_command_on_vm "${vmname}" "test -d /var/log/pcp/pmlogger" ; then echo "WARNING: No PCP data directory on ${vmname}, skipping collection" @@ -328,7 +328,7 @@ collect_pcp_reports_for_vm_offline() { invoke_qemu_script "bash" \ "--vm" "${full_vmname}" \ - "--args" "sudo systemctl stop pmlogger" + "--args" "sudo systemctl stop --now pmlogger" invoke_qemu_script "bash" \ "--vm" "${full_vmname}" \ diff --git a/test/scenario_settings.sh.example b/test/scenario_settings.sh.example index 863ed954a2..391fda56a2 100644 --- a/test/scenario_settings.sh.example +++ b/test/scenario_settings.sh.example @@ -13,6 +13,10 @@ SSH_PUBLIC_KEY=${SSH_PUBLIC_KEY:-${HOME}/.ssh/id_rsa.pub} # local environment where sos can be run manually when necessary. #SKIP_SOS=true +# Disable PCP metrics collection. This can speed up teardown in a +# local environment where PCP can be run manually when necessary. +#SKIP_PCP=true + # Whether to add a VNC graphics console to hosts. This is useful in # local developer settings where cockpit can be used to login to the # host. Set to `true` to enable. Defaults to `false`.