From c3d592d7fdd57023839d051bbab8618538b0dc72 Mon Sep 17 00:00:00 2001 From: Chandan Maurya Date: Wed, 5 Aug 2026 11:24:15 +0530 Subject: [PATCH] Backport node_e2e test migrations to release-4.19 Bulk backport of node_e2e test case migrations from main (OCP 5.0) to release-4.19. These tests were previously backported to release-4.22 (PR #31306, merged), release-4.21 (PR #31426, merged), and release-4.20 (PR #31430, merged). Key adaptations for 4.19: - Created minimal imagepolicy_helpers.go with MCP helper functions since test/extended/imagepolicy/ directory does not exist in 4.19 - Removed ote.Informing() decorators since openshift-tests-extension package is not available in 4.19 - Moved k8s.io/kubelet from indirect to direct dependency Co-authored-by: Cursor --- go.mod | 2 +- .../imagepolicy/imagepolicy_helpers.go | 41 + test/extended/include.go | 1 + test/extended/node/README.md | 77 ++ test/extended/node/node_e2e/OWNERS | 10 + .../node/node_e2e/container_runtime_config.go | 215 ++++ .../node/node_e2e/image_mirror_set.go | 596 +++++++++++ .../node/node_e2e/image_registry_config.go | 123 +++ test/extended/node/node_e2e/initcontainer.go | 155 +++ test/extended/node/node_e2e/netns_cleanup.go | 104 ++ test/extended/node/node_e2e/node.go | 160 +++ test/extended/node/node_e2e/pdb_drain.go | 198 ++++ .../node/node_e2e/probe_termination.go | 285 +++++ test/extended/node/node_mcp_helpers.go | 187 ++++ test/extended/node/node_utils.go | 995 ++++++++++++++++++ test/extended/testdata/bindata.go | 44 + .../testdata/node/node_e2e/pod-dev-fuse.yaml | 20 + .../generated/zz_generated.annotations.go | 28 + 18 files changed, 3240 insertions(+), 1 deletion(-) create mode 100644 test/extended/imagepolicy/imagepolicy_helpers.go create mode 100644 test/extended/node/README.md create mode 100644 test/extended/node/node_e2e/OWNERS create mode 100644 test/extended/node/node_e2e/container_runtime_config.go create mode 100644 test/extended/node/node_e2e/image_mirror_set.go create mode 100644 test/extended/node/node_e2e/image_registry_config.go create mode 100644 test/extended/node/node_e2e/initcontainer.go create mode 100644 test/extended/node/node_e2e/netns_cleanup.go create mode 100644 test/extended/node/node_e2e/node.go create mode 100644 test/extended/node/node_e2e/pdb_drain.go create mode 100644 test/extended/node/node_e2e/probe_termination.go create mode 100644 test/extended/node/node_mcp_helpers.go create mode 100644 test/extended/node/node_utils.go create mode 100644 test/extended/testdata/node/node_e2e/pod-dev-fuse.yaml diff --git a/go.mod b/go.mod index daad484f9c8c..d358be8b64b3 100644 --- a/go.mod +++ b/go.mod @@ -77,6 +77,7 @@ require ( k8s.io/kube-aggregator v0.32.8 k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f k8s.io/kubectl v0.32.8 + k8s.io/kubelet v0.31.1 k8s.io/kubernetes v1.32.8 k8s.io/pod-security-admission v0.32.8 k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d @@ -303,7 +304,6 @@ require ( k8s.io/externaljwt v0.0.0 // indirect k8s.io/kms v0.32.1 // indirect k8s.io/kube-scheduler v0.0.0 // indirect - k8s.io/kubelet v0.31.1 // indirect k8s.io/mount-utils v0.0.0 // indirect k8s.io/sample-apiserver v0.0.0 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect diff --git a/test/extended/imagepolicy/imagepolicy_helpers.go b/test/extended/imagepolicy/imagepolicy_helpers.go new file mode 100644 index 000000000000..950e730781ec --- /dev/null +++ b/test/extended/imagepolicy/imagepolicy_helpers.go @@ -0,0 +1,41 @@ +package imagepolicy + +import ( + "context" + "time" + + o "github.com/onsi/gomega" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + machineconfigclient "github.com/openshift/client-go/machineconfiguration/clientset/versioned" + machineconfighelper "github.com/openshift/origin/test/extended/machine_config" + exutil "github.com/openshift/origin/test/extended/util" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// GetMCPCurrentSpecConfigName returns the current Spec.Configuration.Name for the given MachineConfigPool. +func GetMCPCurrentSpecConfigName(oc *exutil.CLI, pool string) string { + clientSet, err := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig()) + o.Expect(err).NotTo(o.HaveOccurred()) + mcp, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(context.TODO(), pool, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + return mcp.Spec.Configuration.Name +} + +// WaitForMCPConfigSpecChangeAndUpdated waits until Spec.Configuration.Name changes from the provided initial value +// and the MCP reports Updated=true. +func WaitForMCPConfigSpecChangeAndUpdated(oc *exutil.CLI, pool string, initialSpecName string) { + e2e.Logf("Waiting for pool %s to complete", pool) + clientSet, err := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig()) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Eventually(func() bool { + mcp, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(context.TODO(), pool, metav1.GetOptions{}) + if err != nil { + return false + } + if mcp.Status.Configuration.Name == initialSpecName { + return false + } + return machineconfighelper.IsMachineConfigPoolConditionTrue(mcp.Status.Conditions, mcfgv1.MachineConfigPoolUpdated) + }, 20*time.Minute, 10*time.Second).Should(o.BeTrue()) +} diff --git a/test/extended/include.go b/test/extended/include.go index b58d358adeeb..1f1f27ec25f9 100644 --- a/test/extended/include.go +++ b/test/extended/include.go @@ -40,6 +40,7 @@ import ( _ "github.com/openshift/origin/test/extended/machine_config" _ "github.com/openshift/origin/test/extended/machines" _ "github.com/openshift/origin/test/extended/networking" + _ "github.com/openshift/origin/test/extended/node/node_e2e" _ "github.com/openshift/origin/test/extended/node_tuning" _ "github.com/openshift/origin/test/extended/oauth" _ "github.com/openshift/origin/test/extended/olm" diff --git a/test/extended/node/README.md b/test/extended/node/README.md new file mode 100644 index 000000000000..95636564508a --- /dev/null +++ b/test/extended/node/README.md @@ -0,0 +1,77 @@ +# Node E2E Tests + +This directory contains OpenShift end-to-end tests for node-related features. + +## Test Suites + +### Suite: openshift/disruptive-longrunning + +- **node_e2e/container_runtime_config.go** - ContainerRuntimeConfig pidsLimit (OCP-45351) and overlaySize (OCP-46313) - Verifies CTRCFG settings are applied via MCO rollout and reflected on nodes \[Disruptive\] +- **node_e2e/image_mirror_set.go** - ImageDigestMirrorSet and ImageTagMirrorSet (OCP-57401, OCP-70203) - Verifies registries.conf reflects IDMS/ITMS configuration and that ICSP/IDMS/ITMS can coexist \[Disruptive\] \[Serial\] +- **node_e2e/image_registry_config.go** - Container registry config change (OCP-44820) - Verifies search registry update triggers MCO rollout and lands on nodes \[Disruptive\] +- **node_e2e/pdb_drain.go** - PodDisruptionBudget drain blocking (OCP-67564) - Tests that node drain is blocked when PDB has minAvailable=100% with empty selector \[Disruptive\] + +### Suite: openshift/conformance/parallel + +- **node_e2e/initcontainer.go** - Init container restart behavior (OCP-38271) - Verifies init containers do not restart when the exited init container is removed from the node +- **node_e2e/netns_cleanup.go** - Network namespace cleanup (OCP-56266) - Verifies kubelet/CRI-O properly deletes the network namespace when a pod is deleted +- **node_e2e/node.go** - Kubelet log level (KUBELET_LOG_LEVEL), cgroupv2 default validation, and dev fuse enablement in CRI-O (OCP-80983, OCP-70987) +- **node_e2e/probe_termination.go** - Probe-level terminationGracePeriodSeconds (OCP-44493) - Tests configurable termination grace period for liveness and startup probes, including fallback to pod-level config when probe-level is not set + +## Directory Structure + +### Test Files +- All `*.go` files under `node_e2e/` are Ginkgo-based test suites +- Each file focuses on a specific node feature + +### Utility Files +- **node_utils.go** - Shared helper functions for node selection, exec-on-node, and MachineConfigPool rollout waiting +- **node_mcp_helpers.go** - Custom MachineConfigPool creation/cleanup helpers +- **../imagepolicy/imagepolicy_helpers.go** - MachineConfigPool spec-name helpers shared with `node_e2e` tests + +### Test Data +Test fixtures are referenced via `exutil.FixturePath` from: +- `testdata/node/node_e2e/` - Pod fixtures (e.g. dev fuse test pod) + +## Running Tests + +### Running Long-Running Disruptive Tests + +The `openshift/disruptive-longrunning` suite is a general-purpose suite for long-running disruptive tests +across all teams. Node team tests are tagged with `[sig-node]` to identify them. + +To run the entire long-running disruptive test suite on a cluster manually: + +```bash +./openshift-tests run "openshift/disruptive-longrunning" --cluster-stability=Disruptive +``` + +To run only node-specific long-running disruptive tests: + +```bash +./openshift-tests run "openshift/disruptive-longrunning" --dry-run | grep "\[sig-node\]" | ./openshift-tests run -f - --cluster-stability=Disruptive +``` + +## Prerequisites + +- Make sure to set `oc` binary to match the cluster version +- Make sure to set the kubeconfig to point to a live OCP cluster + +## Submitting PRs + +### Adding Tests to `openshift/disruptive-longrunning` + +Before submitting a PR that adds a test to the `openshift/disruptive-longrunning` suite, run the following payload job and include the results in your PR: + +``` +/payload-job periodic-ci-openshift-release-main-nightly-4.19-e2e-aws-disruptive-longrunning +``` + +Useful links for `periodic-ci-openshift-release-main-nightly-4.19-e2e-aws-disruptive-longrunning`: +- [Previous runs (Sippy)](https://sippy.dptools.openshift.org/sippy-ng/jobs/4.19/analysis?filters=%7B%22items%22%3A%5B%7B%22columnField%22%3A%22name%22%2C%22operatorValue%22%3A%22equals%22%2C%22value%22%3A%22periodic-ci-openshift-release-main-nightly-4.19-e2e-aws-disruptive-longrunning%22%7D%5D%7D) +- [Job history for latest runs (Prow)](https://prow.ci.openshift.org/job-history/gs/test-platform-results/logs/periodic-ci-openshift-release-main-nightly-4.19-e2e-aws-disruptive-longrunning) + +## Important Notes + +- Note that dry-run option won't list the test as it does not connect to a live cluster +- Run `make update` if the test data is changed diff --git a/test/extended/node/node_e2e/OWNERS b/test/extended/node/node_e2e/OWNERS new file mode 100644 index 000000000000..28a284c9906d --- /dev/null +++ b/test/extended/node/node_e2e/OWNERS @@ -0,0 +1,10 @@ +reviewers: + - asahay19 + - cpmeadors + - sairameshv + - mrunalp + - BhargaviGudi +approvers: + - cpmeadors + - sairameshv + - mrunalp \ No newline at end of file diff --git a/test/extended/node/node_e2e/container_runtime_config.go b/test/extended/node/node_e2e/container_runtime_config.go new file mode 100644 index 000000000000..9288f93c1179 --- /dev/null +++ b/test/extended/node/node_e2e/container_runtime_config.go @@ -0,0 +1,215 @@ +package node + +import ( + "context" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + machineconfigclient "github.com/openshift/client-go/machineconfiguration/clientset/versioned" + "github.com/openshift/origin/test/extended/imagepolicy" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/utils/ptr" + + nodeutils "github.com/openshift/origin/test/extended/node" + exutil "github.com/openshift/origin/test/extended/util" +) + +var _ = g.Describe("[Suite:openshift/disruptive-longrunning][sig-node][Disruptive] ContainerRuntimeConfig", func() { + var ( + oc = exutil.NewCLIWithoutNamespace("ctrcfg") + ) + + g.BeforeEach(func(ctx context.Context) { + nodeutils.SkipOnMicroShift(oc) + nodeutils.EnsureNodesReady(ctx, oc) + }) + + // Validates that ContainerRuntimeConfig pidsLimit setting is correctly applied + // by MCO to a single worker node and that manual crio.conf edits are overwritten. + //author: cmaurya@redhat.com + g.It("[OTP] Verify pidsLimit and MCO overwrite behavior [OCP-45351]", func() { + ctx := context.Background() + ctrcfgName := "set-pids-limit" + mcpName := "ctrcfg-pids" + + g.By("Get a ready worker node") + workerNode := nodeutils.GetFirstReadyWorkerNode(oc) + o.Expect(workerNode).NotTo(o.BeEmpty(), "no ready worker node found") + err := nodeutils.EnsureNodeHasNoCustomRole(ctx, oc, workerNode) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Make a manual change to crio.conf on worker node") + _, err = nodeutils.ExecOnNodeWithChroot(ctx, oc, workerNode, + "/bin/bash", "-c", `sed -i '/^\[crio\.runtime\]/a log_level = "debug"' /etc/crio/crio.conf`) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to edit crio.conf on node %s", workerNode) + + g.By("Verify the manual crio.conf edit took effect") + editedConf, err := nodeutils.ExecOnNodeWithChroot(ctx, oc, workerNode, "cat", "/etc/crio/crio.conf") + o.Expect(err).NotTo(o.HaveOccurred(), "failed to read crio.conf on node %s", workerNode) + o.Expect(editedConf).To(o.ContainSubstring(`log_level = "debug"`), + "sed edit did not apply: expected log_level = debug in crio.conf") + + mcClient, err := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create machine config client") + + var mcpConfig *nodeutils.CustomMCPConfig + g.DeferCleanup(func() { + cleanupCtx := context.Background() + delErr := oc.MachineConfigurationClient().MachineconfigurationV1().ContainerRuntimeConfigs().Delete( + cleanupCtx, ctrcfgName, metav1.DeleteOptions{}) + if delErr != nil && !apierrors.IsNotFound(delErr) { + e2e.Logf("Warning: failed to delete ContainerRuntimeConfig %s: %v", ctrcfgName, delErr) + } + if err := nodeutils.CleanupCustomMCP(cleanupCtx, mcpConfig); err != nil { + e2e.Logf("WARNING: cleanup had errors: %v", err) + } + }) + + mcpConfig, err = nodeutils.CreateCustomMCPForNode(ctx, oc, mcClient, mcpName, workerNode) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create custom MCP") + + initialSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, mcpName) + + g.By("Create ContainerRuntimeConfig with pidsLimit 2048") + ctrcfg := &mcfgv1.ContainerRuntimeConfig{ + ObjectMeta: metav1.ObjectMeta{Name: ctrcfgName}, + Spec: mcfgv1.ContainerRuntimeConfigSpec{ + MachineConfigPoolSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"machineconfiguration.openshift.io/pool": mcpName}, + }, + ContainerRuntimeConfig: &mcfgv1.ContainerRuntimeConfiguration{ + PidsLimit: ptr.To[int64](2048), + }, + }, + } + _, err = oc.MachineConfigurationClient().MachineconfigurationV1().ContainerRuntimeConfigs().Create( + ctx, ctrcfg, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create ContainerRuntimeConfig") + + g.By("Wait for custom MCP rollout to complete") + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, mcpName, initialSpec) + e2e.Logf("Worker node rolled out successfully") + + g.By("Verify pidsLimit and conmon in crio config on worker node") + var crioConfig string + o.Eventually(func() error { + var execErr error + crioConfig, execErr = nodeutils.ExecOnNodeWithChroot(ctx, oc, workerNode, + "/bin/bash", "-c", "crio config 2>/dev/null") + return execErr + }, 30*time.Second, 5*time.Second).Should(o.Succeed(), "failed to get crio config on node %s", workerNode) + o.Expect(crioConfig).To(o.ContainSubstring("pids_limit = 2048"), "pidsLimit should be 2048") + o.Expect(crioConfig).To(o.ContainSubstring(`conmon = ""`), "conmon should be empty") + o.Expect(crioConfig).NotTo(o.ContainSubstring(`log_level = "debug"`), + "manual crio.conf edit should be overwritten by MCO") + }) + + // Validates that setting overlaySize in ContainerRuntimeConfig is applied to + // storage.conf on a single worker node and the overlay size is reflected inside a container. + //author: cmaurya@redhat.com + g.It("[OTP] Verify overlaySize is applied to node and container [OCP-46313]", func() { + oc.SetupProject() + ctx := context.Background() + ctrcfgName := "ctrcfg-46313" + mcpName := "ctrcfg-overlay" + overlaySize := "9G" + + g.By("Get a ready worker node") + workerNode := nodeutils.GetFirstReadyWorkerNode(oc) + o.Expect(workerNode).NotTo(o.BeEmpty(), "no ready worker node found") + err := nodeutils.EnsureNodeHasNoCustomRole(ctx, oc, workerNode) + o.Expect(err).NotTo(o.HaveOccurred()) + + mcClient, err := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create machine config client") + + var mcpConfig *nodeutils.CustomMCPConfig + g.DeferCleanup(func() { + cleanupCtx := context.Background() + delErr := oc.MachineConfigurationClient().MachineconfigurationV1().ContainerRuntimeConfigs().Delete( + cleanupCtx, ctrcfgName, metav1.DeleteOptions{}) + if delErr != nil && !apierrors.IsNotFound(delErr) { + e2e.Logf("Warning: failed to delete ContainerRuntimeConfig %s: %v", ctrcfgName, delErr) + } + if err := nodeutils.CleanupCustomMCP(cleanupCtx, mcpConfig); err != nil { + e2e.Logf("WARNING: cleanup had errors: %v", err) + } + }) + + mcpConfig, err = nodeutils.CreateCustomMCPForNode(ctx, oc, mcClient, mcpName, workerNode) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create custom MCP") + + initialSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, mcpName) + + g.By("Create ContainerRuntimeConfig with overlaySize " + overlaySize) + quantity := resource.MustParse(overlaySize) + ctrcfg := &mcfgv1.ContainerRuntimeConfig{ + ObjectMeta: metav1.ObjectMeta{Name: ctrcfgName}, + Spec: mcfgv1.ContainerRuntimeConfigSpec{ + MachineConfigPoolSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"machineconfiguration.openshift.io/pool": mcpName}, + }, + ContainerRuntimeConfig: &mcfgv1.ContainerRuntimeConfiguration{ + OverlaySize: &quantity, + }, + }, + } + _, err = oc.MachineConfigurationClient().MachineconfigurationV1().ContainerRuntimeConfigs().Create( + ctx, ctrcfg, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create ContainerRuntimeConfig") + + g.By("Wait for custom MCP rollout to complete") + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, mcpName, initialSpec) + e2e.Logf("Worker node rolled out successfully") + + g.By("Check overlaySize takes effect in storage.conf on worker node") + storageConf, err := nodeutils.ExecOnNodeWithChroot(ctx, oc, workerNode, + "/bin/bash", "-c", "head -n 7 /etc/containers/storage.conf | grep size") + o.Expect(err).NotTo(o.HaveOccurred(), "failed to read storage.conf on node %s", workerNode) + e2e.Logf("storage.conf size line: %s", storageConf) + o.Expect(storageConf).To(o.ContainSubstring(overlaySize), + "storage.conf should contain size = %s", overlaySize) + + g.By("Create a pod on the target node to verify overlay size inside container") + podName := "pod-46313" + ns := oc.Namespace() + err = oc.AsAdmin().WithoutNamespace().Run("run").Args( + podName, "-n", ns, + "--image=quay.io/openshifttest/hello-openshift@sha256:56c354e7885051b6bb4263f9faa58b2c292d44790599b7dde0e49e7c466cf339", + "--restart=Never", + "--overrides", `{"spec":{"nodeName":"`+workerNode+`","securityContext":{"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}},"containers":[{"name":"`+podName+`","image":"quay.io/openshifttest/hello-openshift@sha256:56c354e7885051b6bb4263f9faa58b2c292d44790599b7dde0e49e7c466cf339","command":["/bin/bash","-c","sleep 100000000"],"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}}]}}`, + ).Execute() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create pod") + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", podName, "-n", ns, "--ignore-not-found").Execute() + + g.By("Wait for pod to be running") + err = wait.Poll(5*time.Second, 5*time.Minute, func() (bool, error) { + phase, pollErr := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "pod", podName, "-n", ns, "-o=jsonpath={.status.phase}").Output() + if pollErr != nil { + return false, nil + } + return phase == "Running", nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "pod did not reach Running state") + + g.By("Check overlay filesystem size inside the container") + dfOutput, err := oc.AsAdmin().WithoutNamespace().Run("rsh").Args( + "-n", ns, podName, "/bin/bash", "-c", "df -h / | grep overlay").Output() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to exec df inside pod") + e2e.Logf("overlay df output: %s", dfOutput) + fields := strings.Fields(dfOutput) + o.Expect(len(fields)).To(o.BeNumerically(">=", 2), "unexpected df output format: %s", dfOutput) + actualSize := strings.Split(strings.TrimSuffix(fields[1], "G"), ".")[0] + "G" + o.Expect(actualSize).To(o.Equal(overlaySize), + "overlay filesystem should show %s, got: %s", overlaySize, actualSize) + }) +}) diff --git a/test/extended/node/node_e2e/image_mirror_set.go b/test/extended/node/node_e2e/image_mirror_set.go new file mode 100644 index 000000000000..ef5d2549bdfe --- /dev/null +++ b/test/extended/node/node_e2e/image_mirror_set.go @@ -0,0 +1,596 @@ +package node + +import ( + "context" + "fmt" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + configv1 "github.com/openshift/api/config/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + operatorv1alpha1 "github.com/openshift/api/operator/v1alpha1" + machineconfigclient "github.com/openshift/client-go/machineconfiguration/clientset/versioned" + "github.com/openshift/origin/test/extended/imagepolicy" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + utilrand "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + + nodeutils "github.com/openshift/origin/test/extended/node" + exutil "github.com/openshift/origin/test/extended/util" +) + +// mirrorTestPool isolates mirror-set testing to a single-node custom MCP so rollouts +// do not wait on the full worker and master pools. +type mirrorTestPool struct { + oc *exutil.CLI + mcClient *machineconfigclient.Clientset + PoolName string + NodeName string + nodeLabel string +} + +func newMirrorTestPool(oc *exutil.CLI, ctx context.Context) *mirrorTestPool { + poolName := fmt.Sprintf("mirror-test-%s", utilrand.String(5)) + nodeLabel := fmt.Sprintf("node-role.kubernetes.io/%s", poolName) + + mcClient, err := machineconfigclient.NewForConfig(oc.AdminConfig()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create MachineConfig client") + + nodeName := nodeutils.GetFirstReadyWorkerNode(oc) + o.Expect(nodeName).NotTo(o.BeEmpty(), "no ready worker node found for custom MCP") + err = nodeutils.EnsureNodeHasNoCustomRole(ctx, oc, nodeName) + o.Expect(err).NotTo(o.HaveOccurred()) + + testMCP := &mcfgv1.MachineConfigPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: poolName, + Labels: map[string]string{ + "machineconfiguration.openshift.io/pool": poolName, + }, + }, + Spec: mcfgv1.MachineConfigPoolSpec{ + MachineConfigSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "machineconfiguration.openshift.io/role", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"worker", poolName}, + }, + }, + }, + NodeSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + nodeLabel: "", + }, + }, + }, + } + _, err = mcClient.MachineconfigurationV1().MachineConfigPools().Create(ctx, testMCP, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create custom MachineConfigPool %s", poolName) + + patchData := []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:""}}}`, nodeLabel)) + _, err = oc.AdminKubeClient().CoreV1().Nodes().Patch(ctx, nodeName, types.MergePatchType, patchData, metav1.PatchOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to label node %s for custom MCP", nodeName) + + pool := &mirrorTestPool{ + oc: oc, + mcClient: mcClient, + PoolName: poolName, + NodeName: nodeName, + nodeLabel: nodeLabel, + } + + err = waitForMirrorTestMCPReady(ctx, mcClient, poolName, 10*time.Minute) + o.Expect(err).NotTo(o.HaveOccurred(), "custom MachineConfigPool %s did not become ready", poolName) + e2e.Logf("Custom mirror test pool %s ready on node %s", poolName, nodeName) + + return pool +} + +func (p *mirrorTestPool) currentSpec() string { + return imagepolicy.GetMCPCurrentSpecConfigName(p.oc, p.PoolName) +} + +func (p *mirrorTestPool) waitForRollout(initialSpec string) { + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(p.oc, p.PoolName, initialSpec) +} + +func (p *mirrorTestPool) readRegistriesConf(ctx context.Context) string { + registriesConf, err := nodeutils.ExecOnNodeWithChroot(ctx, p.oc, p.NodeName, "cat", "/etc/containers/registries.conf") + o.Expect(err).NotTo(o.HaveOccurred(), "failed to read registries.conf from node %s", p.NodeName) + return registriesConf +} + +func (p *mirrorTestPool) Teardown() { + ctx := context.Background() + + e2e.Logf("Teardown: removing label %s from node %s", p.nodeLabel, p.NodeName) + removePatch := []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:null}}}`, p.nodeLabel)) + _, patchErr := p.oc.AdminKubeClient().CoreV1().Nodes().Patch(ctx, p.NodeName, types.MergePatchType, removePatch, metav1.PatchOptions{}) + if patchErr != nil && !apierrors.IsNotFound(patchErr) { + e2e.Logf("Warning: failed to remove label from node %s: %v", p.NodeName, patchErr) + } + + e2e.Logf("Teardown: waiting for node %s to transition back to worker pool", p.NodeName) + waitErr := wait.PollUntilContextTimeout(ctx, 15*time.Second, 15*time.Minute, true, func(ctx context.Context) (bool, error) { + currentNode, getErr := p.oc.AdminKubeClient().CoreV1().Nodes().Get(ctx, p.NodeName, metav1.GetOptions{}) + if getErr != nil { + return false, nil + } + currentConfig := currentNode.Annotations["machineconfiguration.openshift.io/currentConfig"] + desiredConfig := currentNode.Annotations["machineconfiguration.openshift.io/desiredConfig"] + return currentConfig != "" && !strings.Contains(currentConfig, p.PoolName) && currentConfig == desiredConfig, nil + }) + if waitErr != nil { + e2e.Logf("Warning: node %s did not transition back to worker pool: %v", p.NodeName, waitErr) + } + + deleteErr := p.mcClient.MachineconfigurationV1().MachineConfigPools().Delete(ctx, p.PoolName, metav1.DeleteOptions{}) + if deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + e2e.Logf("Warning: failed to delete MachineConfigPool %s: %v", p.PoolName, deleteErr) + } + + if deleteErr == nil || apierrors.IsNotFound(deleteErr) { + e2e.Logf("Teardown: waiting for worker MCP to stabilize") + if stabilizeErr := nodeutils.WaitForMCP(ctx, p.mcClient, "worker", 10*time.Minute); stabilizeErr != nil { + e2e.Logf("Warning: worker MCP did not stabilize: %v", stabilizeErr) + } + } +} + +// waitForMirrorTestMCPReady waits for a custom MCP to finish its initial bootstrap. +func waitForMirrorTestMCPReady(ctx context.Context, mcClient *machineconfigclient.Clientset, poolName string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + mcp, err := mcClient.MachineconfigurationV1().MachineConfigPools().Get(ctx, poolName, metav1.GetOptions{}) + if err != nil { + return false, err + } + + updating := false + degraded := false + updated := false + for _, condition := range mcp.Status.Conditions { + switch condition.Type { + case "Updating": + if condition.Status == corev1.ConditionTrue { + updating = true + } + case "Degraded": + if condition.Status == corev1.ConditionTrue { + degraded = true + } + case "Updated": + if condition.Status == corev1.ConditionTrue { + updated = true + } + } + } + + if degraded { + return false, fmt.Errorf("MachineConfigPool %s is degraded", poolName) + } + + isReady := !updating && updated && + mcp.Status.MachineCount > 0 && + mcp.Status.ReadyMachineCount == mcp.Status.MachineCount && + mcp.Spec.Configuration.Name == mcp.Status.Configuration.Name + + return isReady, nil + }) +} + +// pollMCPSpecUnchanged polls the given MCP spec name every 15 seconds for the duration. +// It returns an error if the spec deviates from the baseline, or nil if it stayed stable. +func pollMCPSpecUnchanged(oc *exutil.CLI, pool, baselineSpec string, duration time.Duration) error { + err := wait.PollImmediate(15*time.Second, duration, func() (bool, error) { + if current := imagepolicy.GetMCPCurrentSpecConfigName(oc, pool); current != baselineSpec { + return false, fmt.Errorf("MCP %s spec changed from %s to %s", pool, baselineSpec, current) + } + return false, nil + }) + if err == wait.ErrWaitTimeout { + return nil + } + return err +} + +// author: asahay@redhat.com +var _ = g.Describe("[sig-node][Suite:openshift/disruptive-longrunning][Disruptive][Serial] ImageTagMirrorSet and ImageDigestMirrorSet", func() { + var ( + oc = exutil.NewCLIWithoutNamespace("image-mirror-set") + ) + + g.BeforeEach(func(ctx context.Context) { + nodeutils.SkipOnMicroShift(oc) + nodeutils.EnsureNodesReady(ctx, oc) + }) + + g.It("[OTP] Create ImageDigestMirrorSet and ImageTagMirrorSet and verify registries.conf [OCP-57401]", func(ctx context.Context) { + configClient := oc.AdminConfigClient().ConfigV1() + suffix := utilrand.String(5) + idmsName := fmt.Sprintf("digest-mirror-%s", suffix) + itmsName := fmt.Sprintf("tag-mirror-%s", suffix) + + pool := newMirrorTestPool(oc, ctx) + g.DeferCleanup(pool.Teardown) + + g.By("Step 1: Create an ImageDigestMirrorSet") + idms := &configv1.ImageDigestMirrorSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: idmsName, + }, + Spec: configv1.ImageDigestMirrorSetSpec{ + ImageDigestMirrors: []configv1.ImageDigestMirrors{ + { + Source: "registry.redhat.io/openshift4", + Mirrors: []configv1.ImageMirror{ + "mirror.example.com/redhat", + }, + MirrorSourcePolicy: configv1.AllowContactingSource, + }, + { + Source: "registry.redhat.io/rhel8", + Mirrors: []configv1.ImageMirror{ + "mirror.example.com/rhel8", + }, + MirrorSourcePolicy: configv1.NeverContactSource, + }, + }, + }, + } + + initialSpec := pool.currentSpec() + + createdIDMS, err := configClient.ImageDigestMirrorSets().Create(ctx, idms, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create ImageDigestMirrorSet") + e2e.Logf("ImageDigestMirrorSet %q created successfully", createdIDMS.Name) + + g.DeferCleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + g.By("Cleanup: Delete IDMS and ITMS resources") + cleanupSpec := pool.currentSpec() + if delErr := configClient.ImageTagMirrorSets().Delete(cleanupCtx, itmsName, metav1.DeleteOptions{}); delErr != nil { + e2e.Logf("Warning: failed to delete ImageTagMirrorSet: %v", delErr) + } + if delErr := configClient.ImageDigestMirrorSets().Delete(cleanupCtx, idmsName, metav1.DeleteOptions{}); delErr != nil { + e2e.Logf("Warning: failed to delete ImageDigestMirrorSet: %v", delErr) + } + pool.waitForRollout(cleanupSpec) + }) + + pool.waitForRollout(initialSpec) + e2e.Logf("IDMS MCP rollout complete on custom pool %s", pool.PoolName) + + g.By("Step 2: Create an ImageTagMirrorSet") + itms := &configv1.ImageTagMirrorSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: itmsName, + }, + Spec: configv1.ImageTagMirrorSetSpec{ + ImageTagMirrors: []configv1.ImageTagMirrors{ + { + Source: "registry.access.redhat.com/ubi8/ubi-minimal", + Mirrors: []configv1.ImageMirror{ + "example.io/example/ubi-minimal", + "example.com/example/ubi-minimal", + }, + MirrorSourcePolicy: configv1.AllowContactingSource, + }, + { + Source: "registry.access.redhat.com/ubi8/ubi-minimal-1", + Mirrors: []configv1.ImageMirror{ + "example.io/example/ubi-minimal", + }, + MirrorSourcePolicy: configv1.NeverContactSource, + }, + }, + }, + } + + itmsInitialSpec := pool.currentSpec() + + createdITMS, err := configClient.ImageTagMirrorSets().Create(ctx, itms, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create ImageTagMirrorSet") + e2e.Logf("ImageTagMirrorSet %q created successfully", createdITMS.Name) + + g.By("Step 3: Wait for custom MCP to finish rolling out") + pool.waitForRollout(itmsInitialSpec) + e2e.Logf("Custom MCP %s finished rolling out after ITMS creation", pool.PoolName) + + g.By("Step 4: Verify /etc/containers/registries.conf on the custom pool node") + registriesConf := pool.readRegistriesConf(ctx) + e2e.Logf("registries.conf on %s: read %d bytes, asserting expected entries", pool.NodeName, len(registriesConf)) + + g.By("Verify IDMS entries (digest-only mirrors)") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "registry.redhat.io/openshift4"`), + "registries.conf should contain the IDMS source for openshift4") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "mirror.example.com/redhat"`), + "registries.conf should contain the IDMS mirror for openshift4") + o.Expect(registriesConf).To(o.ContainSubstring(`pull-from-mirror = "digest-only"`), + "registries.conf should have pull-from-mirror set to digest-only for IDMS mirrors") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "registry.redhat.io/rhel8"`), + "registries.conf should contain the IDMS source for rhel8") + + g.By("Verify ITMS entries (tag-only mirrors)") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "registry.access.redhat.com/ubi8/ubi-minimal"`), + "registries.conf should contain the ITMS source for ubi-minimal") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "example.io/example/ubi-minimal"`), + "registries.conf should contain the ITMS mirror location") + o.Expect(registriesConf).To(o.ContainSubstring(`pull-from-mirror = "tag-only"`), + "registries.conf should have pull-from-mirror set to tag-only for ITMS mirrors") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "registry.access.redhat.com/ubi8/ubi-minimal-1"`), + "registries.conf should contain the ITMS source for ubi-minimal-1") + + g.By("Verify NeverContactSource entries are blocked") + o.Expect(registriesConf).To(o.ContainSubstring("location = \"registry.access.redhat.com/ubi8/ubi-minimal-1\"\n blocked = true"), + "registry.access.redhat.com/ubi8/ubi-minimal-1 should be blocked (NeverContactSource)") + o.Expect(registriesConf).To(o.ContainSubstring("location = \"registry.redhat.io/rhel8\"\n blocked = true"), + "registry.redhat.io/rhel8 should be blocked (NeverContactSource)") + }) + + // author: asahay@redhat.com + g.It("[OTP] ICSP and IDMS/ITMS can coexist in cluster [OCP-70203]", func(ctx context.Context) { + configClient := oc.AdminConfigClient().ConfigV1() + operatorClient := oc.AdminOperatorClient().OperatorV1alpha1() + suffix := utilrand.String(5) + icspName1 := fmt.Sprintf("ubi8repo-%s", suffix) + idmsName := fmt.Sprintf("digest-mirror-%s", suffix) + itmsName := fmt.Sprintf("tag-mirror-%s", suffix) + icspName2 := fmt.Sprintf("ubi9repo-%s", suffix) + + pool := newMirrorTestPool(oc, ctx) + g.DeferCleanup(pool.Teardown) + + g.By("Step 1: Create ICSP with digest mirrors for ubi8/ubi-minimal and openshift5") + icsp1 := &operatorv1alpha1.ImageContentSourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: icspName1, + Labels: map[string]string{"e2e-test": "ocp-70203"}, + }, + Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ + RepositoryDigestMirrors: []operatorv1alpha1.RepositoryDigestMirrors{ + { + Source: "registry.access.redhat.com/ubi8/ubi-minimal", + Mirrors: []string{ + "example.io/example/ubi-minimal", + "example.com/example/ubi-minimal", + }, + }, + { + Source: "registry.redhat.io/openshift5", + Mirrors: []string{ + "mirror.example.com/redhat", + }, + }, + }, + }, + } + + initialSpec := pool.currentSpec() + + _, err := operatorClient.ImageContentSourcePolicies().Create(ctx, icsp1, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create ICSP %s", icspName1) + e2e.Logf("ICSP %s created successfully", icspName1) + + g.DeferCleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + g.By("Cleanup: Delete any remaining test resources and wait for custom MCP to settle") + cleanupSpec := pool.currentSpec() + toDelete := false + + for _, name := range []string{icspName1, icspName2} { + if _, getErr := operatorClient.ImageContentSourcePolicies().Get(cleanupCtx, name, metav1.GetOptions{}); getErr == nil { + if delErr := operatorClient.ImageContentSourcePolicies().Delete(cleanupCtx, name, metav1.DeleteOptions{}); delErr == nil { + e2e.Logf("Cleanup: deleted ICSP %s", name) + toDelete = true + } else { + e2e.Logf("Cleanup: warning - failed to delete ICSP %s: %v", name, delErr) + } + } + } + if _, getErr := configClient.ImageTagMirrorSets().Get(cleanupCtx, itmsName, metav1.GetOptions{}); getErr == nil { + if delErr := configClient.ImageTagMirrorSets().Delete(cleanupCtx, itmsName, metav1.DeleteOptions{}); delErr == nil { + e2e.Logf("Cleanup: deleted ITMS %s", itmsName) + toDelete = true + } else { + e2e.Logf("Cleanup: warning - failed to delete ITMS %s: %v", itmsName, delErr) + } + } + if _, getErr := configClient.ImageDigestMirrorSets().Get(cleanupCtx, idmsName, metav1.GetOptions{}); getErr == nil { + if delErr := configClient.ImageDigestMirrorSets().Delete(cleanupCtx, idmsName, metav1.DeleteOptions{}); delErr == nil { + e2e.Logf("Cleanup: deleted IDMS %s", idmsName) + toDelete = true + } else { + e2e.Logf("Cleanup: warning - failed to delete IDMS %s: %v", idmsName, delErr) + } + } + if toDelete { + pool.waitForRollout(cleanupSpec) + } + }) + + g.By("Step 2: Wait for custom MCP rollout after ICSP creation and verify registries.conf") + pool.waitForRollout(initialSpec) + e2e.Logf("Custom MCP %s rollout complete after ICSP creation", pool.PoolName) + + registriesConf := pool.readRegistriesConf(ctx) + e2e.Logf("registries.conf after ICSP creation: read %d bytes, asserting expected entries", len(registriesConf)) + + o.Expect(registriesConf).To(o.ContainSubstring(`location = "registry.access.redhat.com/ubi8/ubi-minimal"`), + "registries.conf should contain ICSP source for ubi8/ubi-minimal") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "example.io/example/ubi-minimal"`), + "registries.conf should contain ICSP mirror example.io/example/ubi-minimal") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "example.com/example/ubi-minimal"`), + "registries.conf should contain ICSP mirror example.com/example/ubi-minimal") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "registry.redhat.io/openshift5"`), + "registries.conf should contain ICSP source for openshift5") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "mirror.example.com/redhat"`), + "registries.conf should contain ICSP mirror mirror.example.com/redhat") + o.Expect(registriesConf).To(o.ContainSubstring(`pull-from-mirror = "digest-only"`), + "registries.conf should have pull-from-mirror = digest-only for ICSP entries") + + g.By("Step 3: Create IDMS with same registry/mirror config as ICSP (AllowContactingSource)") + specBeforeIDMS := pool.currentSpec() + idms := &configv1.ImageDigestMirrorSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: idmsName, + Labels: map[string]string{"e2e-test": "ocp-70203"}, + }, + Spec: configv1.ImageDigestMirrorSetSpec{ + ImageDigestMirrors: []configv1.ImageDigestMirrors{ + { + Source: "registry.access.redhat.com/ubi8/ubi-minimal", + Mirrors: []configv1.ImageMirror{ + "example.io/example/ubi-minimal", + "example.com/example/ubi-minimal", + }, + MirrorSourcePolicy: configv1.AllowContactingSource, + }, + { + Source: "registry.redhat.io/openshift5", + Mirrors: []configv1.ImageMirror{ + "mirror.example.com/redhat", + }, + MirrorSourcePolicy: configv1.AllowContactingSource, + }, + }, + }, + } + _, err = configClient.ImageDigestMirrorSets().Create(ctx, idms, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create IDMS %s", idmsName) + e2e.Logf("IDMS %s created successfully", idmsName) + + g.By("Step 3 (verify): Confirm no new MC was generated after IDMS creation with same config as ICSP") + o.Expect(pollMCPSpecUnchanged(oc, pool.PoolName, specBeforeIDMS, 2*time.Minute)). + NotTo(o.HaveOccurred(), "unexpected MCP rollout after IDMS creation with same config as ICSP") + e2e.Logf("Confirmed: custom MCP %s stable for 2 minutes after IDMS creation", pool.PoolName) + + g.By("Step 4.1: Delete ICSP - IDMS covers the same config so no new MC should be triggered") + specBeforeICSPDelete := pool.currentSpec() + err = operatorClient.ImageContentSourcePolicies().Delete(ctx, icspName1, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to delete ICSP %s", icspName1) + e2e.Logf("ICSP %s deleted successfully", icspName1) + + g.By("Step 4.2: Confirm no new MC was generated after ICSP deletion (IDMS still covers the same config)") + o.Expect(pollMCPSpecUnchanged(oc, pool.PoolName, specBeforeICSPDelete, 2*time.Minute)). + NotTo(o.HaveOccurred(), "unexpected MCP rollout after ICSP deletion when IDMS covers the same config") + e2e.Logf("Confirmed: custom MCP %s stable for 2 minutes after ICSP deletion", pool.PoolName) + + g.By("Step 5: Verify registries.conf is unchanged after ICSP deletion (IDMS maintains same mirror config)") + registriesConfAfterICSPDelete := pool.readRegistriesConf(ctx) + o.Expect(registriesConfAfterICSPDelete).To(o.Equal(registriesConf), + "registries.conf should be unchanged after ICSP deletion when IDMS covers the same mirror config") + e2e.Logf("Confirmed: registries.conf unchanged after ICSP deletion") + + g.By("Step 6: Create ITMS with tag mirrors for ubi9/ubi-minimal (different source from IDMS)") + itmsInitialSpec := pool.currentSpec() + itms := &configv1.ImageTagMirrorSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: itmsName, + Labels: map[string]string{"e2e-test": "ocp-70203"}, + }, + Spec: configv1.ImageTagMirrorSetSpec{ + ImageTagMirrors: []configv1.ImageTagMirrors{ + { + Source: "registry.access.redhat.com/ubi9/ubi-minimal", + Mirrors: []configv1.ImageMirror{ + "example.io/example/ubi-minimal-1", + "example.com/example/ubi-minimal-1", + }, + MirrorSourcePolicy: configv1.AllowContactingSource, + }, + }, + }, + } + _, err = configClient.ImageTagMirrorSets().Create(ctx, itms, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create ITMS %s", itmsName) + e2e.Logf("ITMS %s created successfully", itmsName) + + pool.waitForRollout(itmsInitialSpec) + e2e.Logf("Custom MCP %s rollout complete after ITMS creation", pool.PoolName) + + g.By("Step 7: Verify registries.conf updated with ITMS tag-only entries alongside IDMS digest entries") + registriesConfAfterITMS := pool.readRegistriesConf(ctx) + e2e.Logf("registries.conf after ITMS creation: read %d bytes, asserting expected entries", len(registriesConfAfterITMS)) + + o.Expect(registriesConfAfterITMS).To(o.ContainSubstring(`location = "registry.access.redhat.com/ubi9/ubi-minimal"`), + "registries.conf should contain the ITMS source") + o.Expect(registriesConfAfterITMS).To(o.ContainSubstring(`location = "example.io/example/ubi-minimal-1"`), + "registries.conf should contain ITMS mirror example.io/example/ubi-minimal-1") + o.Expect(registriesConfAfterITMS).To(o.ContainSubstring(`location = "example.com/example/ubi-minimal-1"`), + "registries.conf should contain ITMS mirror example.com/example/ubi-minimal-1") + o.Expect(registriesConfAfterITMS).To(o.ContainSubstring(`pull-from-mirror = "tag-only"`), + "registries.conf should have pull-from-mirror = tag-only for ITMS entries") + o.Expect(registriesConfAfterITMS).To(o.ContainSubstring(`location = "registry.access.redhat.com/ubi8/ubi-minimal"`), + "registries.conf should still contain IDMS entries for ubi8/ubi-minimal") + + g.By("Step 8: Create second ICSP with digest mirrors for registry.example.com/example/myimage") + icsp2InitialSpec := pool.currentSpec() + icsp2 := &operatorv1alpha1.ImageContentSourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: icspName2, + Labels: map[string]string{"e2e-test": "ocp-70203"}, + }, + Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ + RepositoryDigestMirrors: []operatorv1alpha1.RepositoryDigestMirrors{ + { + Source: "registry.example.com/example/myimage", + Mirrors: []string{ + "mirror.example.net/image", + }, + }, + }, + }, + } + _, err = operatorClient.ImageContentSourcePolicies().Create(ctx, icsp2, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create ICSP %s", icspName2) + e2e.Logf("ICSP %s created successfully", icspName2) + + pool.waitForRollout(icsp2InitialSpec) + e2e.Logf("Custom MCP %s rollout complete after second ICSP creation", pool.PoolName) + + g.By("Step 9: Verify registries.conf updated with ICSP2 entries alongside IDMS and ITMS entries") + registriesConfAfterICSP2 := pool.readRegistriesConf(ctx) + e2e.Logf("registries.conf after second ICSP creation: read %d bytes, asserting expected entries", len(registriesConfAfterICSP2)) + + o.Expect(registriesConfAfterICSP2).To(o.ContainSubstring(`location = "registry.example.com/example/myimage"`), + "registries.conf should contain the ICSP2 source") + o.Expect(registriesConfAfterICSP2).To(o.ContainSubstring(`location = "mirror.example.net/image"`), + "registries.conf should contain the ICSP2 mirror") + o.Expect(registriesConfAfterICSP2).To(o.ContainSubstring(`location = "registry.access.redhat.com/ubi8/ubi-minimal"`), + "registries.conf should still contain IDMS entries for ubi8/ubi-minimal") + o.Expect(registriesConfAfterICSP2).To(o.ContainSubstring(`location = "registry.access.redhat.com/ubi9/ubi-minimal"`), + "registries.conf should still contain ITMS entries for ubi9/ubi-minimal") + + g.By("Step 10: Delete IDMS and wait for custom MCP rollout") + idmsDeleteInitialSpec := pool.currentSpec() + err = configClient.ImageDigestMirrorSets().Delete(ctx, idmsName, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to delete IDMS %s", idmsName) + e2e.Logf("IDMS %s deleted successfully", idmsName) + + pool.waitForRollout(idmsDeleteInitialSpec) + e2e.Logf("Custom MCP %s rollout complete after IDMS deletion", pool.PoolName) + + g.By("Step 11: Verify registries.conf - IDMS entries removed, ITMS and ICSP2 entries remain") + registriesConfAfterIDMSDelete := pool.readRegistriesConf(ctx) + e2e.Logf("registries.conf after IDMS deletion: read %d bytes, asserting expected entries", len(registriesConfAfterIDMSDelete)) + + o.Expect(registriesConfAfterIDMSDelete).NotTo(o.ContainSubstring(`location = "registry.access.redhat.com/ubi8/ubi-minimal"`), + "registries.conf should not contain IDMS source registry.access.redhat.com/ubi8/ubi-minimal after IDMS deletion") + o.Expect(registriesConfAfterIDMSDelete).NotTo(o.ContainSubstring(`location = "registry.redhat.io/openshift5"`), + "registries.conf should not contain IDMS source registry.redhat.io/openshift5 after IDMS deletion") + o.Expect(registriesConfAfterIDMSDelete).To(o.ContainSubstring(`location = "registry.access.redhat.com/ubi9/ubi-minimal"`), + "registries.conf should still contain ITMS entries for ubi9/ubi-minimal") + o.Expect(registriesConfAfterIDMSDelete).To(o.ContainSubstring(`location = "registry.example.com/example/myimage"`), + "registries.conf should still contain ICSP2 entries for registry.example.com/example/myimage") + }) +}) diff --git a/test/extended/node/node_e2e/image_registry_config.go b/test/extended/node/node_e2e/image_registry_config.go new file mode 100644 index 000000000000..95dace1af3f8 --- /dev/null +++ b/test/extended/node/node_e2e/image_registry_config.go @@ -0,0 +1,123 @@ +package node + +import ( + "context" + "fmt" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + "github.com/openshift/origin/test/extended/imagepolicy" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + e2e "k8s.io/kubernetes/test/e2e/framework" + + nodeutils "github.com/openshift/origin/test/extended/node" + exutil "github.com/openshift/origin/test/extended/util" + operator "github.com/openshift/origin/test/extended/util/operator" +) + +var _ = g.Describe("[Suite:openshift/disruptive-longrunning][sig-node][Disruptive] Image registry config", func() { + var ( + oc = exutil.NewCLIWithoutNamespace("imgcfg") + ) + + g.BeforeEach(func(ctx context.Context) { + nodeutils.SkipOnMicroShift(oc) + nodeutils.EnsureNodesReady(ctx, oc) + }) + + // Verifies that updating image.config.openshift.io/cluster with a new search + // registry triggers an MCO rollout and the change lands on nodes. + //author: cmaurya@redhat.com + g.It("[OTP] change container registry config [OCP-44820]", func() { + ctx := context.Background() + searchRegistry := "qe.quay.io" + + g.By("Save the original image.config for later restore") + originalImageConfig, err := oc.AdminConfigClient().ConfigV1().Images().Get(ctx, "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get image.config.openshift.io/cluster") + + initialWorkerSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "worker") + initialMasterSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "master") + + g.DeferCleanup(func() { + cleanupCtx := context.Background() + e2e.Logf("Cleanup: restoring original image.config") + restoreErr := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + current, getErr := oc.AdminConfigClient().ConfigV1().Images().Get(cleanupCtx, "cluster", metav1.GetOptions{}) + if getErr != nil { + return getErr + } + current.Spec.RegistrySources = originalImageConfig.Spec.RegistrySources + _, updateErr := oc.AdminConfigClient().ConfigV1().Images().Update(cleanupCtx, current, metav1.UpdateOptions{}) + return updateErr + }) + o.Expect(restoreErr).NotTo(o.HaveOccurred(), + "cleanup failed: could not restore original image.config") + + cleanupWorkerSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "worker") + cleanupMasterSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "master") + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "worker", cleanupWorkerSpec) + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "master", cleanupMasterSpec) + + e2e.Logf("Cleanup: waiting for all cluster operators to settle") + waitErr := operator.WaitForOperatorsToSettle(cleanupCtx, oc.AdminConfigClient(), 10) + o.Expect(waitErr).NotTo(o.HaveOccurred(), + "cluster operators did not settle after restore") + }) + + g.By("Update image.config to add search registry and allowed registries") + err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { + imageConfig, getErr := oc.AdminConfigClient().ConfigV1().Images().Get(ctx, "cluster", metav1.GetOptions{}) + if getErr != nil { + return getErr + } + imageConfig.Spec.RegistrySources.AllowedRegistries = []string{ + "registry.access.redhat.com", "docker.io", "quay.io", searchRegistry, + "image-registry.openshift-image-registry.svc:5000", "quay-proxy.ci.openshift.org", "registry.redhat.io", + } + imageConfig.Spec.RegistrySources.ContainerRuntimeSearchRegistries = []string{ + "registry.access.redhat.com", "docker.io", "quay.io", searchRegistry, + } + _, updateErr := oc.AdminConfigClient().ConfigV1().Images().Update(ctx, imageConfig, metav1.UpdateOptions{}) + return updateErr + }) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to update image.config.openshift.io/cluster") + + g.By("Wait for worker and master MCP rollout to complete") + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "worker", initialWorkerSpec) + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "master", initialMasterSpec) + + g.By("Verify search registries config on a worker node") + workers, err := exutil.GetReadySchedulableWorkerNodes(ctx, oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get ready schedulable worker nodes") + o.Expect(workers).NotTo(o.BeEmpty(), "no ready worker nodes found") + + var registriesConf string + o.Eventually(func() error { + var execErr error + registriesConf, execErr = nodeutils.ExecOnNodeWithChroot(ctx, oc, workers[0].Name, + "cat", "/etc/containers/registries.conf.d/01-image-searchRegistries.conf") + if execErr != nil { + return execErr + } + if !strings.Contains(registriesConf, searchRegistry) { + return fmt.Errorf("search registry %s not yet in config", searchRegistry) + } + return nil + }, 30*time.Second, 5*time.Second).Should(o.Succeed(), + "search registry %s not found in registries config on node %s", searchRegistry, workers[0].Name) + e2e.Logf("Registries config on %s:\n%s", workers[0].Name, registriesConf) + + g.By("Verify policy.json is updated with allowed registries") + policyJSON, err := nodeutils.ExecOnNodeWithChroot(ctx, oc, workers[0].Name, + "cat", "/etc/containers/policy.json") + o.Expect(err).NotTo(o.HaveOccurred(), "failed to read policy.json on node %s", workers[0].Name) + e2e.Logf("policy.json on %s:\n%s", workers[0].Name, policyJSON) + o.Expect(policyJSON).To(o.ContainSubstring(searchRegistry), + "policy.json should contain allowed registry %s", searchRegistry) + }) +}) diff --git a/test/extended/node/node_e2e/initcontainer.go b/test/extended/node/node_e2e/initcontainer.go new file mode 100644 index 000000000000..9d539f722872 --- /dev/null +++ b/test/extended/node/node_e2e/initcontainer.go @@ -0,0 +1,155 @@ +package node + +import ( + "context" + "fmt" + "regexp" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + + nodeutils "github.com/openshift/origin/test/extended/node" + exutil "github.com/openshift/origin/test/extended/util" +) + +var _ = g.Describe("[sig-node] [Jira:Node/Kubelet] NODE initContainer policy,volume,readiness,quota", func() { + defer g.GinkgoRecover() + + var ( + oc = exutil.NewCLI("node-initcontainer") + ) + + // Skip all tests on MicroShift clusters as MachineConfig resources are not available + g.BeforeEach(func(ctx context.Context) { + isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isMicroShift { + g.Skip("Skipping test on MicroShift cluster - MachineConfig resources are not available") + } + + nodeutils.EnsureNodesReady(ctx, oc) + }) + + //author: bgudi@redhat.com + g.It("[OTP] Init containers should not restart when the exited init container is removed from node [OCP-38271]", func() { + g.By("Test for case OCP-38271") + oc.SetupProject() + + podName := "initcon-pod" + namespace := oc.Namespace() + ctx := context.Background() + + g.By("Create a pod with init container") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: namespace, + }, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: "inittest", + Image: "image-registry.openshift-image-registry.svc:5000/openshift/tools:latest", + Command: []string{"/bin/sh", "-ec", "echo running >> /mnt/data/test"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/mnt/data", + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "hello-test", + Image: "image-registry.openshift-image-registry.svc:5000/openshift/tools:latest", + Command: []string{"/bin/sh", "-c", "sleep 3600"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/mnt/data", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + } + + _, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + defer func() { + oc.KubeClient().CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) + }() + + g.By("Check pod status") + err = e2epod.WaitForPodRunningInNamespace(ctx, oc.KubeClient(), pod) + o.Expect(err).NotTo(o.HaveOccurred(), "pod is not running") + + g.By("Get pod and verify init container exited normally") + pod, err = oc.KubeClient().CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + o.Expect(pod.Status.InitContainerStatuses).To(o.ContainElement(o.SatisfyAll( + o.HaveField("Name", "inittest"), + o.HaveField("State.Terminated.ExitCode", o.Equal(int32(0))), + )), "init container 'inittest' should have terminated with exit code 0") + + nodeName := pod.Spec.NodeName + o.Expect(nodeName).NotTo(o.BeEmpty(), "pod node name is empty") + + g.By("Get init container ID from pod status") + var containerID string + for _, status := range pod.Status.InitContainerStatuses { + if status.Name == "inittest" { + containerID = status.ContainerID + break + } + } + o.Expect(containerID).NotTo(o.BeEmpty(), "init container ID is empty") + + // Extract the actual container ID (remove prefix like "cri-o://") + containerIDPattern := regexp.MustCompile(`^[^/]+://(.+)$`) + matches := containerIDPattern.FindStringSubmatch(containerID) + o.Expect(matches).To(o.HaveLen(2), "failed to parse container ID") + actualContainerID := matches[1] + + g.By("Delete init container from node") + output, err := nodeutils.ExecOnNodeWithChroot(ctx, oc, nodeName, "crictl", "rm", actualContainerID) + o.Expect(err).NotTo(o.HaveOccurred(), "fail to delete container") + e2e.Logf("Container deletion output: %s", output) + + g.By("Check init container not restart again") + err = wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) { + pod, err := oc.KubeClient().CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return false, err + } + for _, status := range pod.Status.InitContainerStatuses { + if status.Name == "inittest" { + if status.RestartCount > 0 { + e2e.Logf("Init container restarted, restart count: %d", status.RestartCount) + return true, fmt.Errorf("init container restarted") + } + } + } + e2e.Logf("Init container has not restarted") + return false, nil + }) + o.Expect(err).To(o.Equal(wait.ErrWaitTimeout), "expected timeout while waiting confirms init container did not restart") + }) +}) diff --git a/test/extended/node/node_e2e/netns_cleanup.go b/test/extended/node/node_e2e/netns_cleanup.go new file mode 100644 index 000000000000..a2988c95fe5d --- /dev/null +++ b/test/extended/node/node_e2e/netns_cleanup.go @@ -0,0 +1,104 @@ +package node + +import ( + "context" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + + nodeutils "github.com/openshift/origin/test/extended/node" + exutil "github.com/openshift/origin/test/extended/util" +) + +var _ = g.Describe("[sig-node] [Jira:Node/Kubelet] Network namespace cleanup", func() { + var ( + oc = exutil.NewCLIWithoutNamespace("netns-cleanup") + ) + + g.BeforeEach(func() { + isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isMicroShift { + g.Skip("Skipping test on MicroShift cluster") + } + }) + + //author: bgudi@redhat.com + g.It("[OTP] kubelet/crio will delete netns when a pod is deleted [OCP-56266]", func() { + ctx := context.Background() + oc.SetupProject() + namespace := oc.Namespace() + podName := "pod-56266" + + g.By("Create a test pod") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: namespace, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "hello-openshift", + Image: "image-registry.openshift-image-registry.svc:5000/openshift/tools:latest", + Command: []string{"sleep", "infinity"}, + }, + }, + }, + } + pod, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create pod") + + g.By("Wait for pod to be ready") + err = e2epod.WaitForPodRunningInNamespace(ctx, oc.KubeClient(), pod) + o.Expect(err).NotTo(o.HaveOccurred(), "pod did not become ready") + + g.By("Get pod's node name") + podObj, err := oc.KubeClient().CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get pod") + nodeName := podObj.Spec.NodeName + o.Expect(nodeName).NotTo(o.BeEmpty(), "pod node name is empty") + e2e.Logf("Pod is running on node: %s", nodeName) + + g.By("Get pod's network namespace path") + netNsPath, err := nodeutils.GetPodNetNs(ctx, oc, nodeName, podName) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get pod NetNS") + e2e.Logf("Pod NetNS path: %s", netNsPath) + + g.By("Verify NetNS file exists before pod deletion") + _, err = nodeutils.ExecOnNodeWithChroot(ctx, oc, nodeName, "test", "-e", netNsPath) + o.Expect(err).NotTo(o.HaveOccurred(), "NetNS file does not exist before pod deletion") + + g.By("Delete the pod") + err = oc.KubeClient().CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to delete pod") + + g.By("Wait for pod to be fully deleted") + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { + _, pollErr := oc.KubeClient().CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if apierrors.IsNotFound(pollErr) { + e2e.Logf("Pod deleted successfully") + return true, nil + } + if pollErr != nil { + e2e.Logf("Error checking pod deletion: %v", pollErr) + return false, nil + } + e2e.Logf("Waiting for pod to be deleted") + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "pod was not deleted") + + g.By("Verify that the NetNS file has been cleaned up on the node") + err = nodeutils.CheckNetNsCleaned(ctx, oc, nodeName, netNsPath) + o.Expect(err).NotTo(o.HaveOccurred(), "NetNS file was not cleaned up") + }) +}) diff --git a/test/extended/node/node_e2e/node.go b/test/extended/node/node_e2e/node.go new file mode 100644 index 000000000000..2b6676e93f1c --- /dev/null +++ b/test/extended/node/node_e2e/node.go @@ -0,0 +1,160 @@ +package node + +import ( + "context" + "path/filepath" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + + nodeutils "github.com/openshift/origin/test/extended/node" + exutil "github.com/openshift/origin/test/extended/util" +) + +var _ = g.Describe("[sig-node] [Jira:Node/Kubelet] Kubelet, CRI-O, CPU manager", func() { + var ( + oc = exutil.NewCLIWithoutNamespace("node") + nodeE2EBaseDir = exutil.FixturePath("testdata", "node", "node_e2e") + podDevFuseYAML = filepath.Join(nodeE2EBaseDir, "pod-dev-fuse.yaml") + ) + + g.BeforeEach(func(ctx context.Context) { + nodeutils.SkipOnMicroShift(oc) + nodeutils.EnsureNodesReady(ctx, oc) + }) + + //author: asahay@redhat.com + g.It("[OTP] validate KUBELET_LOG_LEVEL", func(ctx context.Context) { + var kubeservice string + var kubelet string + var err error + + g.By("Polling to check kubelet log level on ready nodes") + waitErr := wait.Poll(10*time.Second, 1*time.Minute, func() (bool, error) { + g.By("Getting all node names in the cluster") + nodeName, nodeErr := oc.AsAdmin().Run("get").Args("nodes", "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(nodeErr).NotTo(o.HaveOccurred()) + e2e.Logf("\nNode Names are %v", nodeName) + nodes := strings.Fields(nodeName) + + for _, node := range nodes { + g.By("Checking if node " + node + " is Ready") + nodeStatus, statusErr := oc.AsAdmin().Run("get").Args("nodes", node, "-o=jsonpath={.status.conditions[?(@.type=='Ready')].status}").Output() + o.Expect(statusErr).NotTo(o.HaveOccurred()) + e2e.Logf("\nNode %s Status is %s\n", node, nodeStatus) + + if nodeStatus == "True" { + g.By("Checking KUBELET_LOG_LEVEL in kubelet.service on node " + node) + kubeservice, err = nodeutils.ExecOnNodeWithChroot(ctx, oc, node, "/bin/bash", "-c", "systemctl show kubelet.service | grep KUBELET_LOG_LEVEL") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Checking kubelet process for --v=2 flag on node " + node) + kubelet, err = nodeutils.ExecOnNodeWithChroot(ctx, oc, node, "/bin/bash", "-c", "ps aux | grep [k]ubelet") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verifying KUBELET_LOG_LEVEL is set and kubelet is running with --v=2") + if strings.Contains(kubeservice, "KUBELET_LOG_LEVEL") && strings.Contains(kubelet, "--v=2") { + e2e.Logf("KUBELET_LOG_LEVEL is 2.\n") + return true, nil + } else { + e2e.Logf("KUBELET_LOG_LEVEL is not 2.\n") + return false, nil + } + } else { + e2e.Logf("\nNode %s is not Ready, Skipping\n", node) + } + } + return false, nil + }) + + if waitErr != nil { + e2e.Logf("Kubelet Log level is:\n %v\n", kubeservice) + e2e.Logf("Running Process of kubelet are:\n %v\n", kubelet) + } + o.Expect(waitErr).NotTo(o.HaveOccurred(), "KUBELET_LOG_LEVEL is not expected, timed out") + }) + + //author: cmaurya@redhat.com + g.It("[OTP] validate cgroupv2 is default [OCP-80983]", func(ctx context.Context) { + g.By("Check cgroup version on all Ready worker nodes") + nodeNames, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-l", "node-role.kubernetes.io/worker", "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + workers := strings.Fields(nodeNames) + o.Expect(workers).NotTo(o.BeEmpty(), "No worker nodes found") + + for _, worker := range workers { + nodeStatus, err := oc.AsAdmin().Run("get").Args("nodes", worker, "-o=jsonpath={.status.conditions[?(@.type=='Ready')].status}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if nodeStatus != "True" { + e2e.Logf("Skipping worker node %s (not Ready)", worker) + continue + } + cgroupV, err := nodeutils.ExecOnNodeWithChroot(ctx, oc, worker, "/bin/bash", "-c", "stat -c %T -f /sys/fs/cgroup") + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("cgroup version on node %s: [%v]", worker, cgroupV) + o.Expect(cgroupV).To(o.ContainSubstring("cgroup2fs"), "Node %s does not have cgroupv2", worker) + } + + g.By("Changing cgroup from v2 to v1 should result in error") + output, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("nodes.config.openshift.io", "cluster", "-p", `{"spec": {"cgroupMode": "v1"}}`, "--type=merge").Output() + o.Expect(err).Should(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("spec.cgroupMode: Unsupported value: \"v1\": supported values: \"v2\", \"\"")) + }) + + //author: cmaurya@redhat.com + g.It("[OTP] Allow dev fuse by default in CRI-O [OCP-70987]", func(ctx context.Context) { + podName := "pod-devfuse" + ns := "devfuse-test" + + // Skip on runc: io.kubernetes.cri-o.Devices annotation is only in crun's allowed_annotations. + // We query crio config directly as ContainerRuntimeConfig API misses platform-default runc. + g.By("Skip if the default runtime is runc") + node, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "nodes", "-l", "node-role.kubernetes.io/worker", "-o=jsonpath={.items[0].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(node).NotTo(o.BeEmpty()) + runtime, err := nodeutils.ExecOnNodeWithChroot(ctx, oc, node, "/bin/bash", "-c", + "crio status config 2>/dev/null | awk -F'\"' '/default_runtime/{print $2}'") + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.TrimSpace(runtime) == "runc" { + g.Skip("Skipping: not applicable to runc runtime") + } + + g.By("Create a test namespace") + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("namespace", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("namespace", ns, "--ignore-not-found").Execute() + + g.By("Create a pod with dev fuse annotation") + err = oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", podDevFuseYAML, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Wait for pod to be ready") + err = wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) { + status, pollErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", podName, "-n", ns, "-o=jsonpath={.status.conditions[?(@.type=='Ready')].status}").Output() + if pollErr != nil { + e2e.Logf("Error polling pod status: %v", pollErr) + return false, nil + } + return status == "True", nil + }) + if err != nil { + podStatus, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", podName, "-n", ns, "-o=jsonpath={.status}").Output() + e2e.Logf("Pod status on timeout: %s", podStatus) + } + o.Expect(err).NotTo(o.HaveOccurred(), "pod did not become ready") + + g.By("Check /dev/fuse is mounted inside the pod") + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args(podName, "-n", ns, "--", "stat", "/dev/fuse").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("/dev/fuse mount output: %s", output) + o.Expect(output).To(o.ContainSubstring("fuse"), "dev fuse is not mounted inside pod") + }) +}) + +// ImageTagMirrorSet and ImageDigestMirrorSet tests (OCP-57401, OCP-70203) live in image_mirror_set.go. diff --git a/test/extended/node/node_e2e/pdb_drain.go b/test/extended/node/node_e2e/pdb_drain.go new file mode 100644 index 000000000000..efd83c2c5d0b --- /dev/null +++ b/test/extended/node/node_e2e/pdb_drain.go @@ -0,0 +1,198 @@ +package node + +import ( + "context" + "fmt" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/utils/ptr" + + nodeutils "github.com/openshift/origin/test/extended/node" + exutil "github.com/openshift/origin/test/extended/util" + "github.com/openshift/origin/test/extended/util/operator" +) + +var _ = g.Describe("[Suite:openshift/disruptive-longrunning][sig-node][Disruptive] PodDisruptionBudget", func() { + var ( + oc = exutil.NewCLIWithoutNamespace("pdb-drain") + ) + + g.BeforeEach(func(ctx context.Context) { + nodeutils.SkipOnMicroShift(oc) + nodeutils.EnsureNodesReady(ctx, oc) + }) + + //author: bgudi@redhat.com + g.It("[OTP] Node's drain should block when PodDisruptionBudget minAvailable equals 100 percentage and selector is empty [OCP-67564]", func() { + ctx := context.Background() + + // Skip on SNO/External topologies where there might not be dedicated worker nodes + infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(ctx, "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get cluster infrastructure") + if infra.Status.ControlPlaneTopology == "SingleReplica" || infra.Status.ControlPlaneTopology == "External" { + g.Skip("Skipping on SNO/External topology - requires dedicated worker nodes") + } + + oc.SetupProject() + namespace := oc.Namespace() + + g.By("Get a worker node to schedule pods on") + workers, err := exutil.GetReadySchedulableWorkerNodes(ctx, oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get worker nodes") + o.Expect(workers).NotTo(o.BeEmpty(), "no ready schedulable worker nodes found") + workerNode := workers[0].Name + e2e.Logf("Selected worker node: %s", workerNode) + + g.By("Create 6 pods on the selected worker node") + numPods := 6 + podBaseName := "pdb-drain-test-pod" + for i := 0; i < numPods; i++ { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", podBaseName, i), + Namespace: namespace, + Labels: map[string]string{ + "app": "pdb-drain-test", + }, + }, + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{ + "kubernetes.io/hostname": workerNode, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + Containers: []corev1.Container{ + { + Name: "test-container", + Image: "quay.io/openshifttest/hello-openshift@sha256:4200f438cf2e9446f6bcff9d67ceea1f69ed07a2f83363b7fb52529f7ddd8a83", + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + }, + }, + }, + }, + } + _, err = oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("failed to create pod %d", i)) + } + + g.By("Wait for all pods to be ready") + err = wait.PollUntilContextTimeout(ctx, 3*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + podList, pollErr := oc.KubeClient().CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=pdb-drain-test", + }) + if pollErr != nil { + e2e.Logf("Error getting pods: %v", pollErr) + return false, nil + } + readyPods := 0 + for _, pod := range podList.Items { + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { + readyPods++ + break + } + } + } + if readyPods == numPods { + e2e.Logf("All %d pods are ready", readyPods) + return true, nil + } + e2e.Logf("Waiting for pods to be ready: %d/%d", readyPods, numPods) + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "pods did not become ready") + + g.By("Create PodDisruptionBudget with 100% minAvailable and empty selector") + pdb := &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pdb-drain-test", + Namespace: namespace, + }, + Spec: policyv1.PodDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "100%", + }, + Selector: &metav1.LabelSelector{}, + }, + } + _, err = oc.KubeClient().PolicyV1().PodDisruptionBudgets(namespace).Create(ctx, pdb, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create PodDisruptionBudget") + g.DeferCleanup(oc.KubeClient().PolicyV1().PodDisruptionBudgets(namespace).Delete, context.Background(), "pdb-drain-test", metav1.DeleteOptions{}) + + g.By("Verify all test pods are on the selected worker node") + podList, err := oc.KubeClient().CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=pdb-drain-test", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get pods") + podsOnWorker := 0 + for _, pod := range podList.Items { + if pod.Spec.NodeName == workerNode { + podsOnWorker++ + } + } + o.Expect(podsOnWorker).To(o.Equal(numPods), "not all pods are on the selected worker node") + + g.By("Make sure that PDB's DisruptionAllowed condition is False") + var pdbStatus string + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 30*time.Second, true, func(pollCtx context.Context) (bool, error) { + var pollErr error + pdbStatus, pollErr = oc.AsAdmin().WithoutNamespace().Run("get").Args("poddisruptionbudget", "pdb-drain-test", "-n", namespace, "-o=jsonpath={.status.conditions[?(@.type==\"DisruptionAllowed\")].status}").Output() + if pollErr != nil { + e2e.Logf("Error getting PDB status: %v", pollErr) + return false, nil + } + if pdbStatus != "" { + return true, nil + } + e2e.Logf("Waiting for PDB DisruptionAllowed condition to appear") + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "PDB DisruptionAllowed condition not found") + o.Expect(pdbStatus).Should(o.Equal("False"), "PDB DisruptionAllowed should be False") + + g.By("Drain the selected worker node") + g.DeferCleanup(func() { + cleanupCtx := context.Background() + err := operator.WaitForOperatorsToSettle(cleanupCtx, oc.AdminConfigClient(), 10) + o.Expect(err).NotTo(o.HaveOccurred(), "cluster operators failed to return to available state after node drain") + }) + g.DeferCleanup(oc.AsAdmin().WithoutNamespace().Run("adm").Args("uncordon", workerNode).Execute) + + out, err := oc.AsAdmin().WithoutNamespace().Run("adm").Args("drain", workerNode, "--ignore-daemonsets", "--delete-emptydir-data", "--force", "--timeout=30s").Output() + o.Expect(err).To(o.HaveOccurred(), "drain operation should have been blocked but it wasn't") + o.Expect(strings.Contains(out, "Cannot evict pod as it would violate the pod's disruption budget")).Should(o.BeTrue(), "drain output missing PDB violation error message") + o.Expect(strings.Contains(out, "There are pending nodes to be drained")).Should(o.BeTrue(), "drain output missing pending nodes error message") + + g.By("Verify that test pods remain on the node after failed drain") + podsAfterDrain, err := oc.KubeClient().CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=pdb-drain-test", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get pods after drain attempt") + podsStillOnWorker := 0 + for _, pod := range podsAfterDrain.Items { + if pod.Spec.NodeName == workerNode { + podsStillOnWorker++ + } + } + o.Expect(podsStillOnWorker).To(o.Equal(numPods), "all test pods should still be on the worker node") + }) +}) diff --git a/test/extended/node/node_e2e/probe_termination.go b/test/extended/node/node_e2e/probe_termination.go new file mode 100644 index 000000000000..48fe5aa6ae82 --- /dev/null +++ b/test/extended/node/node_e2e/probe_termination.go @@ -0,0 +1,285 @@ +package node + +import ( + "context" + "fmt" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/utils/ptr" + + nodeutils "github.com/openshift/origin/test/extended/node" + exutil "github.com/openshift/origin/test/extended/util" + "github.com/openshift/origin/test/extended/util/image" +) + +// Tolerance applied around the expected termination time: -minToleranceSec for timing +// precision, +maxToleranceSec for container cleanup/restart overhead. +const ( + minToleranceSec = 3 + maxToleranceSec = 10 +) + +var _ = g.Describe("[sig-node] Probe configuration", func() { + var ( + oc = exutil.NewCLIWithoutNamespace("probe-termination") + ) + + g.BeforeEach(func(ctx context.Context) { + isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isMicroShift { + g.Skip("Skipping test on MicroShift cluster") + } + + nodeutils.EnsureNodesReady(ctx, oc) + }) + + //author: bgudi@redhat.com + g.It("[OTP] Liveness probe should respect probe-level terminationGracePeriodSeconds [OCP-44493]", func() { + ctx := context.Background() + + oc.SetupProject() + namespace := oc.Namespace() + + g.By("Create pod with liveness probe having probe-level terminationGracePeriodSeconds=10s") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "liveness-probe-level", + Namespace: namespace, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: ptr.To[int64](60), + Containers: []corev1.Container{ + { + Name: "test", + Image: image.ShellImage(), + Command: []string{"sh", "-c", "sleep 100000000"}, + Ports: []corev1.ContainerPort{ + {ContainerPort: 8080}, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(8080), + }, + }, + InitialDelaySeconds: 5, + FailureThreshold: 1, + PeriodSeconds: 60, + TerminationGracePeriodSeconds: ptr.To[int64](10), + }, + }, + }, + }, + } + + _, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create liveness probe pod") + + g.By("Verify probe-level terminationGracePeriodSeconds is honored (10s)") + expectedSec := 10 + timeDiff, err := verifyProbeTermination(ctx, oc, namespace, "liveness-probe-level", "test", expectedSec) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to observe probe termination and restart") + assertTerminationWithinTolerance(timeDiff, expectedSec) + }) + + //author: bgudi@redhat.com + g.It("[OTP] Startup probe should respect probe-level terminationGracePeriodSeconds [OCP-44493]", func() { + ctx := context.Background() + + oc.SetupProject() + namespace := oc.Namespace() + + g.By("Create pod with startup probe having probe-level terminationGracePeriodSeconds=10s") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "startup-probe-level", + Namespace: namespace, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: ptr.To[int64](60), + Containers: []corev1.Container{ + { + Name: "teststartup", + Image: image.ShellImage(), + Command: []string{"sh", "-c", "sleep 100000000"}, + Ports: []corev1.ContainerPort{ + {ContainerPort: 8080}, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(8080), + }, + }, + InitialDelaySeconds: 5, + FailureThreshold: 1, + PeriodSeconds: 60, + TerminationGracePeriodSeconds: ptr.To[int64](10), + }, + }, + }, + }, + } + + _, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create startup probe pod") + + g.By("Verify probe-level terminationGracePeriodSeconds is honored (10s)") + expectedSec := 10 + timeDiff, err := verifyProbeTermination(ctx, oc, namespace, "startup-probe-level", "teststartup", expectedSec) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to observe probe termination and restart") + assertTerminationWithinTolerance(timeDiff, expectedSec) + }) + + //author: bgudi@redhat.com + g.It("[OTP] Liveness probe should fall back to pod-level terminationGracePeriodSeconds when probe-level is not set [OCP-44493]", func() { + ctx := context.Background() + + oc.SetupProject() + namespace := oc.Namespace() + + g.By("Create pod with liveness probe without probe-level terminationGracePeriodSeconds") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "liveness-pod-level", + Namespace: namespace, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: ptr.To[int64](60), + Containers: []corev1.Container{ + { + Name: "test", + Image: image.ShellImage(), + Command: []string{"sh", "-c", "sleep 100000000"}, + Ports: []corev1.ContainerPort{ + {ContainerPort: 8080}, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(8080), + }, + }, + InitialDelaySeconds: 5, + FailureThreshold: 1, + PeriodSeconds: 60, + // No TerminationGracePeriodSeconds - should use pod-level (60s) + }, + }, + }, + }, + } + + _, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create liveness probe pod without probe-level termination") + + g.By("Verify pod-level terminationGracePeriodSeconds is used (60s)") + expectedSec := 60 + timeDiff, err := verifyProbeTermination(ctx, oc, namespace, "liveness-pod-level", "test", expectedSec) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to observe probe termination and restart") + assertTerminationWithinTolerance(timeDiff, expectedSec) + }) +}) + +// assertTerminationWithinTolerance asserts that timeDiff falls within the tolerance window +// around expectedSec (see minToleranceSec/maxToleranceSec). +func assertTerminationWithinTolerance(timeDiff, expectedSec int) { + minSec := expectedSec - minToleranceSec + maxSec := expectedSec + maxToleranceSec + o.Expect(timeDiff).To(o.BeNumerically(">=", minSec), fmt.Sprintf("time difference %ds is less than expected minimum %ds", timeDiff, minSec)) + o.Expect(timeDiff).To(o.BeNumerically("<=", maxSec), fmt.Sprintf("time difference %ds is greater than expected maximum %ds", timeDiff, maxSec)) +} + +// verifyProbeTermination measures the time between the "Killing" event (kill decision, via +// FirstTimestamp) and the container's restart (via pod.Status, not a "Started" event, since +// LastTimestamp on repeated events is unreliable). Restart is gated to RestartCount==1 so the +// two signals always refer to the same cycle. Returns the difference in seconds. +func verifyProbeTermination(ctx context.Context, oc *exutil.CLI, namespace, podName, containerName string, expectedTerminationSec int) (int, error) { + var timeDiff int + // Timeout needs to account for: pod start (~30s) + probe period (60s) + termination (up to 60s) + restart (~30s) = ~3 minutes minimum + // Use 6 minutes to be safe for tests with 60s termination grace period + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 6*time.Minute, true, func(ctx context.Context) (bool, error) { + pod, err := oc.KubeClient().CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + e2e.Logf("Error getting pod %s: %v", podName, err) + return false, nil + } + + status := e2epod.FindContainerStatusInPod(pod, containerName) + if status == nil { + e2e.Logf("Waiting for container status of %q to appear", containerName) + return false, nil + } + + // Gate strictly on the first restart so the "Killing" event's FirstTimestamp below + // is guaranteed to correspond to the same cycle as this restart, not a later one. + if status.RestartCount != 1 { + e2e.Logf("Waiting for the first restart of %q (restartCount=%d)", containerName, status.RestartCount) + return false, nil + } + + if status.State.Running == nil { + e2e.Logf("Waiting for container %q to be running again after restart", containerName) + return false, nil + } + + events, err := oc.KubeClient().CoreV1().Events(namespace).List(ctx, metav1.ListOptions{ + FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.kind=Pod,reason=Killing", podName), + }) + if err != nil { + e2e.Logf("Error listing events for pod %s: %v", podName, err) + return false, nil + } + + killingEvent := findProbeKillingEvent(events, containerName) + if killingEvent == nil { + e2e.Logf("Waiting for probe-failure (Killing) event for container %q", containerName) + return false, nil + } + + killedAt := killingEvent.FirstTimestamp.Time + startedAt := status.State.Running.StartedAt.Time + if !startedAt.After(killedAt) { + // The status hasn't caught up yet (e.g. reporting a stale running instance); keep polling. + e2e.Logf("Waiting for a fresh Running state after kill decision at %v", killedAt) + return false, nil + } + + timeDiff = int(startedAt.Sub(killedAt).Seconds()) + e2e.Logf("Container %q: probe failure detected at %v, restarted at %v, time difference: %d seconds (expected: %d, -%d/+%ds tolerance)", + containerName, killedAt, startedAt, timeDiff, expectedTerminationSec, minToleranceSec, maxToleranceSec) + + return true, nil + }) + if err != nil { + return 0, fmt.Errorf("failed to observe probe termination and restart for container %q: %w", containerName, err) + } + return timeDiff, nil +} + +// findProbeKillingEvent finds the "Killing" event kubelet records when a probe failure +// triggers container termination. The message format is "Container failed +// liveness/startup probe, will be restarted" (see kuberuntime_manager.go). +func findProbeKillingEvent(events *corev1.EventList, containerName string) *corev1.Event { + for i := range events.Items { + event := &events.Items[i] + if strings.Contains(event.Message, containerName) && strings.Contains(event.Message, "failed") && strings.Contains(event.Message, "probe") { + return event + } + } + return nil +} diff --git a/test/extended/node/node_mcp_helpers.go b/test/extended/node/node_mcp_helpers.go new file mode 100644 index 000000000000..394254fe6210 --- /dev/null +++ b/test/extended/node/node_mcp_helpers.go @@ -0,0 +1,187 @@ +package node + +import ( + "context" + "fmt" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + + corev1 "k8s.io/api/core/v1" + + machineconfigv1 "github.com/openshift/api/machineconfiguration/v1" + machineconfigclient "github.com/openshift/client-go/machineconfiguration/clientset/versioned" + exutil "github.com/openshift/origin/test/extended/util" +) + +var standardNodeRoles = map[string]bool{ + "worker": true, + "control-plane": true, + "master": true, +} + +func NodeHasCustomRole(node corev1.Node) (string, bool) { + const prefix = "node-role.kubernetes.io/" + for label := range node.Labels { + if strings.HasPrefix(label, prefix) { + role := strings.TrimPrefix(label, prefix) + if !standardNodeRoles[role] { + return role, true + } + } + } + return "", false +} + +func EnsureNodeHasNoCustomRole(ctx context.Context, oc *exutil.CLI, nodeName string) error { + node, err := oc.AdminKubeClient().CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get node %s: %w", nodeName, err) + } + if role, found := NodeHasCustomRole(*node); found { + return fmt.Errorf("node %s already has custom role %q; another test's MCP cleanup may not have completed", nodeName, role) + } + return nil +} + +// CustomMCPConfig holds state needed to clean up a custom MCP created by CreateCustomMCPForNode. +type CustomMCPConfig struct { + Name string + NodeName string + MCClient *machineconfigclient.Clientset + KubeClient *exutil.CLI +} + +// CreateCustomMCPForNode labels a node and creates a custom MCP targeting it. +func CreateCustomMCPForNode(ctx context.Context, oc *exutil.CLI, mcClient *machineconfigclient.Clientset, mcpName, nodeName string) (*CustomMCPConfig, error) { + config := &CustomMCPConfig{ + Name: mcpName, + NodeName: nodeName, + MCClient: mcClient, + KubeClient: oc, + } + + if err := EnsureNodeHasNoCustomRole(ctx, oc, nodeName); err != nil { + return nil, err + } + + nodeLabel := fmt.Sprintf("node-role.kubernetes.io/%s", mcpName) + + framework.Logf("Labeling node %s with %s", nodeName, nodeLabel) + patchData := []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:""}}}`, nodeLabel)) + _, err := oc.AdminKubeClient().CoreV1().Nodes().Patch(ctx, nodeName, types.MergePatchType, patchData, metav1.PatchOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to label node %s: %w", nodeName, err) + } + + framework.Logf("Creating custom MachineConfigPool %s", mcpName) + mcp := &machineconfigv1.MachineConfigPool{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "machineconfiguration.openshift.io/v1", + Kind: "MachineConfigPool", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: mcpName, + Labels: map[string]string{ + "machineconfiguration.openshift.io/pool": mcpName, + "pools.operator.machineconfiguration.openshift.io/" + mcpName: "", + }, + }, + Spec: machineconfigv1.MachineConfigPoolSpec{ + MachineConfigSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "machineconfiguration.openshift.io/role", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"worker", mcpName}, + }, + }, + }, + NodeSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + nodeLabel: "", + }, + }, + }, + } + + _, err = mcClient.MachineconfigurationV1().MachineConfigPools().Create(ctx, mcp, metav1.CreateOptions{}) + if err != nil { + framework.Logf("MCP creation failed, removing node label") + unlabelPatchData := []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:null}}}`, nodeLabel)) + _, _ = oc.AdminKubeClient().CoreV1().Nodes().Patch(ctx, nodeName, types.MergePatchType, unlabelPatchData, metav1.PatchOptions{}) + return nil, fmt.Errorf("failed to create MachineConfigPool %s: %w", mcpName, err) + } + + framework.Logf("Waiting for custom MachineConfigPool %s to be ready", mcpName) + err = WaitForMCP(ctx, mcClient, mcpName, 5*time.Minute) + if err != nil { + return config, fmt.Errorf("MachineConfigPool %s did not become ready: %w", mcpName, err) + } + + framework.Logf("Custom MachineConfigPool %s created successfully", mcpName) + return config, nil +} + +// CleanupCustomMCP removes the node label, deletes the custom MCP, and waits for the worker MCP to stabilize. +func CleanupCustomMCP(ctx context.Context, config *CustomMCPConfig) error { + if config == nil { + return nil + } + + nodeLabel := fmt.Sprintf("node-role.kubernetes.io/%s", config.Name) + var cleanupErrors []error + + framework.Logf("Removing node label %s from node %s", nodeLabel, config.NodeName) + patchData := []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:null}}}`, nodeLabel)) + _, err := config.KubeClient.AdminKubeClient().CoreV1().Nodes().Patch(ctx, config.NodeName, types.MergePatchType, patchData, metav1.PatchOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + cleanupErrors = append(cleanupErrors, fmt.Errorf("failed to remove label from node %s: %w", config.NodeName, err)) + } + + if err == nil || apierrors.IsNotFound(err) { + framework.Logf("Waiting for node %s to transition back to worker pool", config.NodeName) + transitionErr := wait.PollUntilContextTimeout(ctx, 10*time.Second, 7*time.Minute, true, func(ctx context.Context) (bool, error) { + node, getErr := config.KubeClient.AdminKubeClient().CoreV1().Nodes().Get(ctx, config.NodeName, metav1.GetOptions{}) + if apierrors.IsNotFound(getErr) { + return true, nil + } + if getErr != nil { + return false, nil + } + currentConfig := node.Annotations["machineconfiguration.openshift.io/currentConfig"] + desiredConfig := node.Annotations["machineconfiguration.openshift.io/desiredConfig"] + isWorkerConfig := currentConfig != "" && !strings.Contains(currentConfig, config.Name) && currentConfig == desiredConfig + return isWorkerConfig, nil + }) + if transitionErr != nil { + cleanupErrors = append(cleanupErrors, fmt.Errorf("node %s did not transition back to worker pool: %w", config.NodeName, transitionErr)) + } + } + + framework.Logf("Deleting custom MachineConfigPool %s", config.Name) + deleteErr := config.MCClient.MachineconfigurationV1().MachineConfigPools().Delete(ctx, config.Name, metav1.DeleteOptions{}) + if deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + cleanupErrors = append(cleanupErrors, fmt.Errorf("failed to delete MachineConfigPool %s: %w", config.Name, deleteErr)) + } + + if deleteErr == nil || apierrors.IsNotFound(deleteErr) { + framework.Logf("Waiting for worker MCP to stabilize after custom MCP deletion") + waitErr := WaitForMCP(ctx, config.MCClient, "worker", 10*time.Minute) + if waitErr != nil && !apierrors.IsNotFound(waitErr) { + cleanupErrors = append(cleanupErrors, fmt.Errorf("worker MCP did not stabilize: %w", waitErr)) + } + } + + if len(cleanupErrors) > 0 { + return fmt.Errorf("cleanup completed with errors: %v", cleanupErrors) + } + + framework.Logf("Custom MachineConfigPool %s cleaned up successfully", config.Name) + return nil +} diff --git a/test/extended/node/node_utils.go b/test/extended/node/node_utils.go new file mode 100644 index 000000000000..7ce343742b81 --- /dev/null +++ b/test/extended/node/node_utils.go @@ -0,0 +1,995 @@ +package node + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "os" + "regexp" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" + "k8s.io/kubernetes/test/e2e/framework" + + machineconfigv1 "github.com/openshift/api/machineconfiguration/v1" + machineconfigclient "github.com/openshift/client-go/machineconfiguration/clientset/versioned" + exutil "github.com/openshift/origin/test/extended/util" +) + +// SkipOnMicroShift skips the current test if the cluster is MicroShift. +func SkipOnMicroShift(oc *exutil.CLI) { + isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isMicroShift { + g.Skip("Skipping test on MicroShift cluster") + } +} + +// getNodesByLabel returns nodes matching the specified label selector +func getNodesByLabel(ctx context.Context, oc *exutil.CLI, labelSelector string) ([]corev1.Node, error) { + nodes, err := oc.AdminKubeClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + return nil, err + } + return nodes.Items, nil +} + +// getControlPlaneNodes returns all control plane nodes in the cluster +func getControlPlaneNodes(ctx context.Context, oc *exutil.CLI) ([]corev1.Node, error) { + // Try master label first (OpenShift uses this) + nodes, err := getNodesByLabel(ctx, oc, "node-role.kubernetes.io/master") + if err != nil { + return nil, err + } + if len(nodes) > 0 { + return nodes, nil + } + + // Fallback to control-plane label (upstream Kubernetes uses this) + return getNodesByLabel(ctx, oc, "node-role.kubernetes.io/control-plane") +} + +// getKubeletConfigFromNode retrieves the kubelet configuration from a specific node +func getKubeletConfigFromNode(ctx context.Context, oc *exutil.CLI, nodeName string) (*kubeletconfigv1beta1.KubeletConfiguration, error) { + // Use the node proxy API to get configz + configzPath := fmt.Sprintf("/api/v1/nodes/%s/proxy/configz", nodeName) + + data, err := oc.AdminKubeClient().CoreV1().RESTClient().Get().AbsPath(configzPath).DoRaw(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get configz from node %s: %w", nodeName, err) + } + + // Parse the JSON response + var configzResponse struct { + KubeletConfig *kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"` + } + + if err := json.Unmarshal(data, &configzResponse); err != nil { + return nil, fmt.Errorf("failed to unmarshal configz response: %w", err) + } + + if configzResponse.KubeletConfig == nil { + return nil, fmt.Errorf("kubeletconfig is nil in response") + } + + return configzResponse.KubeletConfig, nil +} + +// getPureWorkerNodesFromCluster returns worker nodes that are not also control plane nodes. +func getPureWorkerNodesFromCluster(ctx context.Context, oc *exutil.CLI) ([]corev1.Node, error) { + workers, err := getNodesByLabel(ctx, oc, "node-role.kubernetes.io/worker") + if err != nil { + return nil, err + } + pureWorkers := getPureWorkerNodes(workers) + if len(pureWorkers) == 0 { + return nil, fmt.Errorf("no pure worker nodes available") + } + return pureWorkers, nil +} + +// getPureWorkerNodes returns worker nodes that are not also control plane nodes. +// On SNO clusters, the single node has both worker and control-plane roles, +// so it should be validated as a control plane node (failSwapOn=true), not as a worker. +func getPureWorkerNodes(nodes []corev1.Node) []corev1.Node { + var pureWorkers []corev1.Node + for _, node := range nodes { + _, hasControlPlane := node.Labels["node-role.kubernetes.io/control-plane"] + _, hasMaster := node.Labels["node-role.kubernetes.io/master"] + if hasControlPlane || hasMaster { + framework.Logf("Skipping node %s (control-plane role)", node.Name) + continue + } + if role, found := NodeHasCustomRole(node); found { + framework.Logf("Skipping node %s (already has custom role %q)", node.Name, role) + continue + } + pureWorkers = append(pureWorkers, node) + } + return pureWorkers +} + +const ( + // DebugNamespace is the namespace for debug pods + DebugNamespace = "openshift-machine-config-operator" + // cnvNamespace is the namespace for CNV operator + cnvNamespace = "openshift-cnv" + // cnvOperatorGroup is the name of the CNV operator group + cnvOperatorGroup = "kubevirt-hyperconverged-group" + // cnvSubscription is the name of the CNV subscription + cnvSubscription = "hco-operatorhub" + // cnvHyperConverged is the name of the HyperConverged CR + cnvHyperConverged = "kubevirt-hyperconverged" + // cnvNodeLabel is the label for CNV-schedulable nodes + cnvNodeLabel = "kubevirt.io/schedulable" +) + +// GVRs for CNV resources +var ( + subscriptionGVR = schema.GroupVersionResource{ + Group: "operators.coreos.com", + Version: "v1alpha1", + Resource: "subscriptions", + } + operatorGroupGVR = schema.GroupVersionResource{ + Group: "operators.coreos.com", + Version: "v1", + Resource: "operatorgroups", + } + hyperConvergedGVR = schema.GroupVersionResource{ + Group: "hco.kubevirt.io", + Version: "v1beta1", + Resource: "hyperconvergeds", + } + csvGVR = schema.GroupVersionResource{ + Group: "operators.coreos.com", + Version: "v1alpha1", + Resource: "clusterserviceversions", + } + mcpGVR = schema.GroupVersionResource{ + Group: "machineconfiguration.openshift.io", + Version: "v1", + Resource: "machineconfigpools", + } +) + +// getCNVWorkerNodeName returns the name of a worker node with CNV label (kubevirt.io/schedulable=true) +func getCNVWorkerNodeName(ctx context.Context, oc *exutil.CLI) string { + // First try to get nodes with CNV schedulable label + nodes, err := getNodesByLabel(ctx, oc, "kubevirt.io/schedulable=true") + if err == nil && len(nodes) > 0 { + // Randomly select a node from the available CNV nodes + return nodes[rand.Intn(len(nodes))].Name + } + + // Fallback to any worker node + nodes, err = getNodesByLabel(ctx, oc, "node-role.kubernetes.io/worker") + if err != nil || len(nodes) == 0 { + return "" + } + // Randomly select a node from available worker nodes + return nodes[rand.Intn(len(nodes))].Name +} + +func execOnNodeWithDebug(ctx context.Context, oc *exutil.CLI, nodeName string, timeout time.Duration, args []string) (string, error) { + timeoutCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + execCmd, stdOutBuf, stdErrBuf, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args(args...).Background() + if err != nil { + return "", err + } + + type result struct { + err error + } + resultCh := make(chan result, 1) + + go func() { + resultCh <- result{err: execCmd.Wait()} + }() + + select { + case res := <-resultCh: + stdOut := strings.TrimSpace(stdOutBuf.String()) + stdErr := strings.TrimSpace(stdErrBuf.String()) + if res.err != nil { + return stdOut, fmt.Errorf("oc debug failed: %w\nStdErr: %s", res.err, stdErr) + } + return stdOut, nil + case <-timeoutCtx.Done(): + var killErr error + if execCmd.Process != nil { + killErr = execCmd.Process.Kill() + } + if ctx.Err() != nil { + return "", fmt.Errorf("oc debug command canceled on node %s: %w", nodeName, ctx.Err()) + } + if killErr != nil { + return "", fmt.Errorf("oc debug command timed out after %v on node %s; failed to stop debug process: %w", timeout, nodeName, killErr) + } + return "", fmt.Errorf("oc debug command timed out after %v on node %s (cleanup likely hung)", timeout, nodeName) + } +} + +func ExecOnNodeWithChroot(ctx context.Context, oc *exutil.CLI, nodeName string, cmd ...string) (string, error) { + args := append([]string{"node/" + nodeName, "-n" + DebugNamespace, "--", "chroot", "/host"}, cmd...) + return execOnNodeWithDebug(ctx, oc, nodeName, 2*time.Minute, args) +} + +func ExecOnNodeWithNsenter(ctx context.Context, oc *exutil.CLI, nodeName string, cmd ...string) (string, error) { + nsenterCmd := append([]string{"nsenter", "-a", "-t", "1"}, cmd...) + args := append([]string{"node/" + nodeName, "-n" + DebugNamespace, "--"}, nsenterCmd...) + return execOnNodeWithDebug(ctx, oc, nodeName, 2*time.Minute, args) +} + +// createDropInFile creates a drop-in configuration file on the specified node +func createDropInFile(ctx context.Context, oc *exutil.CLI, nodeName, filePath, content string) error { + escapedContent := strings.ReplaceAll(content, "'", "'\\''") + cmd := fmt.Sprintf("echo '%s' > %s && chmod 644 %s", escapedContent, filePath, filePath) + _, err := ExecOnNodeWithChroot(ctx, oc, nodeName, "sh", "-c", cmd) + return err +} + +// removeDropInFile removes a drop-in configuration file from the specified node +func removeDropInFile(ctx context.Context, oc *exutil.CLI, nodeName, filePath string) error { + _, err := ExecOnNodeWithChroot(ctx, oc, nodeName, "rm", "-f", filePath) + return err +} + +// loadConfigFromFile reads kubelet configuration from a YAML file +func loadConfigFromFile(path string) string { + data, err := os.ReadFile(path) + if err != nil { + framework.Failf("Failed to read config file %s: %v", path, err) + } + return string(data) +} + +// restartKubeletOnNode restarts the kubelet service on the specified node +// Retries on transient network errors which are common on real clusters +func restartKubeletOnNode(ctx context.Context, oc *exutil.CLI, nodeName string) error { + const maxAttempts = 3 + var lastErr error + for attempt := 0; attempt < maxAttempts; attempt++ { + _, err := ExecOnNodeWithChroot(ctx, oc, nodeName, "systemctl", "restart", "kubelet") + if err == nil { + return nil + } + lastErr = err + if !isTransientNetworkError(err) { + return fmt.Errorf("failed to restart kubelet on %s: %w", nodeName, err) + } + if attempt == maxAttempts-1 { + break + } + backoff := time.Duration((attempt+1)*5) * time.Second + framework.Logf("Attempt %d/%d to restart kubelet on %s failed: %v; retrying in %s", + attempt+1, maxAttempts, nodeName, err, backoff) + timer := time.NewTimer(backoff) + select { + case <-ctx.Done(): + timer.Stop() + return fmt.Errorf("context canceled while restarting kubelet on %s: %w", nodeName, ctx.Err()) + case <-timer.C: + } + } + return fmt.Errorf("failed to restart kubelet on %s after %d attempts: %w", nodeName, maxAttempts, lastErr) +} + +// isTransientNetworkError checks if the error is a transient network error worth retrying +func isTransientNetworkError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + transientErrors := []string{ + "connection refused", + "connection reset", + "connection timed out", + "i/o timeout", + } + for _, transientErr := range transientErrors { + if strings.Contains(errStr, transientErr) { + return true + } + } + return false +} + +// waitForNodeToBeReady waits for a node to become Ready +func waitForNodeToBeReady(ctx context.Context, oc *exutil.CLI, nodeName string) { + o.Eventually(func() bool { + node, err := oc.AdminKubeClient().CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return false + } + return isNodeInReadyState(node) + }, 5*time.Minute, 10*time.Second).Should(o.BeTrue(), "Node %s should become Ready", nodeName) +} + +// isNodeInReadyState checks if a node is in Ready condition +func isNodeInReadyState(node *corev1.Node) bool { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +// cleanupDropInAndRestartKubelet removes the drop-in file and restarts kubelet +func cleanupDropInAndRestartKubelet(ctx context.Context, oc *exutil.CLI, nodeName, filePath string) { + framework.Logf("Removing drop-in file: %s", filePath) + removeDropInFile(ctx, oc, nodeName, filePath) + framework.Logf("Restarting kubelet on node: %s", nodeName) + restartKubeletOnNode(ctx, oc, nodeName) + framework.Logf("Waiting for node to be ready...") + waitForNodeToBeReady(ctx, oc, nodeName) +} + +// ============================================================================ +// CNV Operator Installation/Uninstallation Functions +// ============================================================================ + +// isCNVInstalled checks if CNV operator is installed +func isCNVInstalled(ctx context.Context, oc *exutil.CLI) bool { + // Check if CNV namespace exists + _, err := oc.AdminKubeClient().CoreV1().Namespaces().Get(ctx, cnvNamespace, metav1.GetOptions{}) + if err != nil { + return false + } + + // Check if HyperConverged CR exists + dynamicClient := oc.AdminDynamicClient() + _, err = dynamicClient.Resource(hyperConvergedGVR).Namespace(cnvNamespace).Get(ctx, cnvHyperConverged, metav1.GetOptions{}) + return err == nil +} + +// installCNVOperator installs the CNV operator and creates HyperConverged CR +func installCNVOperator(ctx context.Context, oc *exutil.CLI) error { + framework.Logf("Installing CNV operator...") + + dynamicClient := oc.AdminDynamicClient() + + // Step 1: Create CNV namespace with Pod Security labels + // CNV requires privileged access for networking DaemonSets (bridge plugins, etc.) + framework.Logf("Creating namespace %s with Pod Security labels", cnvNamespace) + + podSecurityLabels := map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + "security.openshift.io/scc.podSecurityLabelSync": "false", + } + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: cnvNamespace, + Labels: podSecurityLabels, + }, + } + + _, err := oc.AdminKubeClient().CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + // Namespace exists, update it to ensure Pod Security labels are set + framework.Logf("Namespace %s already exists, updating Pod Security labels", cnvNamespace) + existingNs, getErr := oc.AdminKubeClient().CoreV1().Namespaces().Get(ctx, cnvNamespace, metav1.GetOptions{}) + if getErr != nil { + return fmt.Errorf("failed to get existing namespace %s: %w", cnvNamespace, getErr) + } + if existingNs.Labels == nil { + existingNs.Labels = make(map[string]string) + } + for k, v := range podSecurityLabels { + existingNs.Labels[k] = v + } + _, updateErr := oc.AdminKubeClient().CoreV1().Namespaces().Update(ctx, existingNs, metav1.UpdateOptions{}) + if updateErr != nil { + return fmt.Errorf("failed to update namespace %s with Pod Security labels: %w", cnvNamespace, updateErr) + } + } else { + return fmt.Errorf("failed to create namespace %s: %w", cnvNamespace, err) + } + } + + // Step 2: Create OperatorGroup + framework.Logf("Creating OperatorGroup %s", cnvOperatorGroup) + operatorGroup := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "operators.coreos.com/v1", + "kind": "OperatorGroup", + "metadata": map[string]interface{}{ + "name": cnvOperatorGroup, + "namespace": cnvNamespace, + }, + "spec": map[string]interface{}{ + "targetNamespaces": []interface{}{ + cnvNamespace, + }, + }, + }, + } + _, err = dynamicClient.Resource(operatorGroupGVR).Namespace(cnvNamespace).Create(ctx, operatorGroup, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create OperatorGroup: %w", err) + } + + // Step 3: Create Subscription + framework.Logf("Creating Subscription %s", cnvSubscription) + subscription := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "operators.coreos.com/v1alpha1", + "kind": "Subscription", + "metadata": map[string]interface{}{ + "name": cnvSubscription, + "namespace": cnvNamespace, + }, + "spec": map[string]interface{}{ + "channel": "stable", + "installPlanApproval": "Automatic", + "name": "kubevirt-hyperconverged", + "source": "redhat-operators", + "sourceNamespace": "openshift-marketplace", + // Note: startingCSV can be specified for specific versions + // "startingCSV": "kubevirt-hyperconverged-operator.v4.17.0", + }, + }, + } + _, err = dynamicClient.Resource(subscriptionGVR).Namespace(cnvNamespace).Create(ctx, subscription, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create Subscription: %w", err) + } + + // Step 4: Wait for CSV to be ready + framework.Logf("Waiting for CNV operator to be installed...") + err = waitForCNVOperatorReady(ctx, oc) + if err != nil { + return fmt.Errorf("CNV operator installation failed: %w", err) + } + + // Step 5: Create HyperConverged CR + framework.Logf("Creating HyperConverged CR %s", cnvHyperConverged) + hyperConverged := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "hco.kubevirt.io/v1beta1", + "kind": "HyperConverged", + "metadata": map[string]interface{}{ + "name": cnvHyperConverged, + "namespace": cnvNamespace, + }, + "spec": map[string]interface{}{ + "BareMetalPlatform": true, + "infra": map[string]interface{}{}, + "workloads": map[string]interface{}{}, + }, + }, + } + _, err = dynamicClient.Resource(hyperConvergedGVR).Namespace(cnvNamespace).Create(ctx, hyperConverged, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create HyperConverged CR: %w", err) + } + + // Step 6: Wait for HyperConverged to be ready + framework.Logf("Waiting for HyperConverged to be ready...") + err = waitForHyperConvergedReady(ctx, oc) + if err != nil { + return fmt.Errorf("HyperConverged failed to become ready: %w", err) + } + + // Step 7: Label worker nodes for CNV + framework.Logf("Labeling worker nodes for CNV...") + err = labelWorkerNodesForCNV(ctx, oc) + if err != nil { + framework.Logf("Warning: failed to label nodes for CNV: %v", err) + } + + // Step 8: Wait for MCP rollout to complete (if any MachineConfigs were applied) + framework.Logf("Checking MCP rollout status...") + mcClient, err := machineconfigclient.NewForConfig(oc.AdminConfig()) + if err != nil { + return fmt.Errorf("failed to create MC client for MCP check: %w", err) + } + + err = WaitForMCP(ctx, mcClient, "worker", 15*time.Minute) + if err != nil { + return fmt.Errorf("MCP rollout failed after CNV installation: %w", err) + } + + framework.Logf("CNV operator installed successfully") + return nil +} + +// waitForCNVOperatorReady waits for the CNV operator CSV to be in Succeeded phase +func waitForCNVOperatorReady(ctx context.Context, oc *exutil.CLI) error { + dynamicClient := oc.AdminDynamicClient() + + return wait.PollUntilContextTimeout(ctx, 15*time.Second, 15*time.Minute, true, func(ctx context.Context) (bool, error) { + // List CSVs in the namespace + csvList, err := dynamicClient.Resource(csvGVR).Namespace(cnvNamespace).List(ctx, metav1.ListOptions{}) + if err != nil { + framework.Logf("Error listing CSVs: %v", err) + return false, nil + } + + for _, csv := range csvList.Items { + name := csv.GetName() + if strings.Contains(name, "kubevirt-hyperconverged") { + phase, found, err := unstructured.NestedString(csv.Object, "status", "phase") + if err != nil || !found { + framework.Logf("CSV %s phase not found yet", name) + return false, nil + } + framework.Logf("CSV %s phase: %s", name, phase) + if phase == "Succeeded" { + return true, nil + } + } + } + return false, nil + }) +} + +// waitForHyperConvergedReady waits for the HyperConverged CR to be ready +func waitForHyperConvergedReady(ctx context.Context, oc *exutil.CLI) error { + dynamicClient := oc.AdminDynamicClient() + + return wait.PollUntilContextTimeout(ctx, 15*time.Second, 15*time.Minute, true, func(ctx context.Context) (bool, error) { + hc, err := dynamicClient.Resource(hyperConvergedGVR).Namespace(cnvNamespace).Get(ctx, cnvHyperConverged, metav1.GetOptions{}) + if err != nil { + framework.Logf("Error getting HyperConverged: %v", err) + return false, nil + } + + conditions, found, err := unstructured.NestedSlice(hc.Object, "status", "conditions") + if err != nil || !found { + framework.Logf("HyperConverged conditions not found yet") + return false, nil + } + + for _, cond := range conditions { + condition, ok := cond.(map[string]interface{}) + if !ok { + continue + } + condType, _, _ := unstructured.NestedString(condition, "type") + condStatus, _, _ := unstructured.NestedString(condition, "status") + + if condType == "Available" && condStatus == "True" { + framework.Logf("HyperConverged is Available") + return true, nil + } + } + framework.Logf("Waiting for HyperConverged to become Available...") + return false, nil + }) +} + +type waitMCPOptions struct { + machineCount *int32 + allowDegraded bool +} + +// WaitMCPWithMachineCount waits until the pool has the exact machine count. +func WaitMCPWithMachineCount(count int32) func(*waitMCPOptions) { + return func(o *waitMCPOptions) { + o.machineCount = &count + } +} + +// WaitMCPAllowDegraded tolerates transient Degraded/RenderDegraded during polling instead of +// failing immediately. The pool must still become fully healthy (!degraded, !renderDegraded, +// updated, all machines ready) before this wait succeeds. +func WaitMCPAllowDegraded() func(*waitMCPOptions) { + return func(o *waitMCPOptions) { + o.allowDegraded = true + } +} + +// WaitForMCP waits for a MachineConfigPool to be ready (not updating, updated, and all machines ready). +// By default it returns an error immediately if the MCP becomes degraded. +func WaitForMCP(ctx context.Context, mcClient *machineconfigclient.Clientset, poolName string, timeout time.Duration, opts ...func(*waitMCPOptions)) error { + options := waitMCPOptions{} + for _, opt := range opts { + opt(&options) + } + + framework.Logf("Waiting for MCP %s to be ready (timeout: %v)...", poolName, timeout) + + poolSeen := false + return wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + mcp, err := mcClient.MachineconfigurationV1().MachineConfigPools().Get(ctx, poolName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + // Only treat NotFound as success when draining a pool we have previously observed. + // A typo in poolName must not pass silently on the first poll. + if poolSeen && options.machineCount != nil && *options.machineCount == 0 { + framework.Logf("MachineConfigPool %s no longer exists after draining to 0 machines", poolName) + return true, nil + } + } + return false, err + } + poolSeen = true + + updating := false + degraded := false + renderDegraded := false + ready := false + + for _, condition := range mcp.Status.Conditions { + switch condition.Type { + case machineconfigv1.MachineConfigPoolUpdating: + if condition.Status == corev1.ConditionTrue { + updating = true + } + case machineconfigv1.MachineConfigPoolDegraded: + if condition.Status == corev1.ConditionTrue { + degraded = true + } + case machineconfigv1.MachineConfigPoolRenderDegraded: + if condition.Status == corev1.ConditionTrue { + renderDegraded = true + } + case machineconfigv1.MachineConfigPoolUpdated: + if condition.Status == corev1.ConditionTrue { + ready = true + } + } + } + + if !options.allowDegraded { + if degraded { + return false, fmt.Errorf("MachineConfigPool %s is degraded", poolName) + } + if renderDegraded { + return false, fmt.Errorf("MachineConfigPool %s render is degraded", poolName) + } + } + + // Drained pool: all nodes removed and conditions stable before cleanup continues. + if options.machineCount != nil && *options.machineCount == 0 { + isDrained := mcp.Status.MachineCount == 0 && + mcp.Status.ReadyMachineCount == 0 && + !updating && !degraded && !renderDegraded + if isDrained { + framework.Logf("MachineConfigPool %s drained: 0 machines, not updating/degraded", poolName) + return true, nil + } + framework.Logf("MachineConfigPool %s waiting to drain: updating=%v degraded=%v renderDegraded=%v machines=%d/%d", + poolName, updating, degraded, renderDegraded, mcp.Status.ReadyMachineCount, mcp.Status.MachineCount) + return false, nil + } + + isReady := !updating && !degraded && !renderDegraded && ready && + mcp.Status.MachineCount > 0 && mcp.Status.ReadyMachineCount == mcp.Status.MachineCount + if options.machineCount != nil { + isReady = isReady && mcp.Status.MachineCount == *options.machineCount + } + + if isReady { + framework.Logf("MachineConfigPool %s is ready: %d/%d machines ready", + poolName, mcp.Status.ReadyMachineCount, mcp.Status.MachineCount) + } else { + framework.Logf("MachineConfigPool %s not ready yet: updating=%v degraded=%v renderDegraded=%v updated=%v machines=%d/%d", + poolName, updating, degraded, renderDegraded, ready, mcp.Status.ReadyMachineCount, mcp.Status.MachineCount) + } + + return isReady, nil + }) +} + +// getWorkerGeneratedKubeletMC finds and returns the highest numbered worker-generated-kubelet MachineConfig. +// KubeletConfig changes affect the highest numbered config, so we return that one. +func getWorkerGeneratedKubeletMC(ctx context.Context, mcClient *machineconfigclient.Clientset) (*machineconfigv1.MachineConfig, error) { + mcList, err := mcClient.MachineconfigurationV1().MachineConfigs().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + + var highestMC *machineconfigv1.MachineConfig + for i := range mcList.Items { + if strings.Contains(mcList.Items[i].Name, "worker-generated-kubelet") { + if highestMC == nil || mcList.Items[i].Name > highestMC.Name { + highestMC = &mcList.Items[i] + } + } + } + + if highestMC == nil { + return nil, fmt.Errorf("worker-generated-kubelet MachineConfig not found") + } + + return highestMC, nil +} + +// labelWorkerNodesForCNV labels all worker nodes with kubevirt.io/schedulable=true +func labelWorkerNodesForCNV(ctx context.Context, oc *exutil.CLI) error { + framework.Logf("Labeling worker nodes for CNV...") + + nodes, err := getNodesByLabel(ctx, oc, "node-role.kubernetes.io/worker") + if err != nil { + return fmt.Errorf("failed to get worker nodes: %w", err) + } + + for _, node := range nodes { + framework.Logf("Labeling node %s with %s=true", node.Name, cnvNodeLabel) + nodeCopy := node.DeepCopy() + if nodeCopy.Labels == nil { + nodeCopy.Labels = make(map[string]string) + } + nodeCopy.Labels[cnvNodeLabel] = "true" + _, err := oc.AdminKubeClient().CoreV1().Nodes().Update(ctx, nodeCopy, metav1.UpdateOptions{}) + if err != nil { + framework.Logf("Warning: failed to label node %s: %v", node.Name, err) + } + } + + return nil +} + +// unlabelWorkerNodesForCNV removes the kubevirt.io/schedulable label from worker nodes +func unlabelWorkerNodesForCNV(ctx context.Context, oc *exutil.CLI) error { + framework.Logf("Removing CNV labels from worker nodes...") + + nodes, err := getNodesByLabel(ctx, oc, cnvNodeLabel+"=true") + if err != nil { + return fmt.Errorf("failed to get CNV-labeled nodes: %w", err) + } + + for _, node := range nodes { + framework.Logf("Removing label %s from node %s", cnvNodeLabel, node.Name) + nodeCopy := node.DeepCopy() + delete(nodeCopy.Labels, cnvNodeLabel) + _, err := oc.AdminKubeClient().CoreV1().Nodes().Update(ctx, nodeCopy, metav1.UpdateOptions{}) + if err != nil { + framework.Logf("Warning: failed to unlabel node %s: %v", node.Name, err) + } + } + + return nil +} + +// uninstallCNVOperator uninstalls the CNV operator and all related resources +func uninstallCNVOperator(ctx context.Context, oc *exutil.CLI) error { + framework.Logf("Uninstalling CNV operator...") + + dynamicClient := oc.AdminDynamicClient() + + // Step 1: Delete HyperConverged CR + // Give the operator 3 minutes to clean up gracefully; if the CR is still stuck, + // strip its finalizers so deletion can proceed. + framework.Logf("Deleting HyperConverged CR %s", cnvHyperConverged) + err := dynamicClient.Resource(hyperConvergedGVR).Namespace(cnvNamespace).Delete(ctx, cnvHyperConverged, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + framework.Logf("Warning: failed to delete HyperConverged CR: %v", err) + } + + if err == nil || !apierrors.IsNotFound(err) { + framework.Logf("Waiting for HyperConverged CR to be deleted...") + finalizersStripped := false + startTime := time.Now() + _ = wait.PollUntilContextTimeout(ctx, 10*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + _, getErr := dynamicClient.Resource(hyperConvergedGVR).Namespace(cnvNamespace).Get(ctx, cnvHyperConverged, metav1.GetOptions{}) + if apierrors.IsNotFound(getErr) { + return true, nil + } + + if !finalizersStripped && time.Since(startTime) > 3*time.Minute { + framework.Logf("HyperConverged CR still present after 3m, stripping finalizers") + patch := []byte(`{"metadata":{"finalizers":[]}}`) + _, patchErr := dynamicClient.Resource(hyperConvergedGVR).Namespace(cnvNamespace).Patch( + ctx, cnvHyperConverged, types.MergePatchType, patch, metav1.PatchOptions{}) + if patchErr != nil && !apierrors.IsNotFound(patchErr) { + framework.Logf("Warning: failed to strip finalizers from HyperConverged CR: %v", patchErr) + } + finalizersStripped = true + } + + framework.Logf("Waiting for HyperConverged to be deleted...") + return false, nil + }) + } + + // Step 2: Delete Subscription + framework.Logf("Deleting Subscription %s", cnvSubscription) + err = dynamicClient.Resource(subscriptionGVR).Namespace(cnvNamespace).Delete(ctx, cnvSubscription, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + framework.Logf("Warning: failed to delete Subscription: %v", err) + } + + // Step 3: Delete all CSVs in the namespace + framework.Logf("Deleting CSVs in namespace %s", cnvNamespace) + csvList, err := dynamicClient.Resource(csvGVR).Namespace(cnvNamespace).List(ctx, metav1.ListOptions{}) + if err == nil { + for _, csv := range csvList.Items { + _ = dynamicClient.Resource(csvGVR).Namespace(cnvNamespace).Delete(ctx, csv.GetName(), metav1.DeleteOptions{}) + } + } + + // Step 4: Delete OperatorGroup + framework.Logf("Deleting OperatorGroup %s", cnvOperatorGroup) + err = dynamicClient.Resource(operatorGroupGVR).Namespace(cnvNamespace).Delete(ctx, cnvOperatorGroup, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + framework.Logf("Warning: failed to delete OperatorGroup: %v", err) + } + + // Step 5: Remove node labels + framework.Logf("Removing CNV node labels...") + _ = unlabelWorkerNodesForCNV(ctx, oc) + + // Step 6: Delete namespace + framework.Logf("Deleting namespace %s", cnvNamespace) + err = oc.AdminKubeClient().CoreV1().Namespaces().Delete(ctx, cnvNamespace, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + framework.Logf("Warning: failed to delete namespace: %v", err) + } + + // Wait for namespace to be deleted + framework.Logf("Waiting for namespace to be deleted...") + _ = wait.PollUntilContextTimeout(ctx, 10*time.Second, 10*time.Minute, true, func(ctx context.Context) (bool, error) { + _, err := oc.AdminKubeClient().CoreV1().Namespaces().Get(ctx, cnvNamespace, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, nil + } + framework.Logf("Waiting for namespace %s to be deleted...", cnvNamespace) + return false, nil + }) + + // Step 7: Wait for MCP rollout to complete (if any MachineConfigs were removed) + framework.Logf("Checking MCP rollout status after CNV uninstallation...") + mcClient, err := machineconfigclient.NewForConfig(oc.AdminConfig()) + if err != nil { + framework.Logf("Warning: failed to create MC client for MCP check: %v", err) + } else { + err = WaitForMCP(ctx, mcClient, "worker", 15*time.Minute) + if err != nil { + framework.Logf("Warning: MCP rollout check failed: %v", err) + } + } + + framework.Logf("CNV operator uninstalled successfully") + return nil +} + +// ensureDropInDirectoryExists creates the drop-in directory on worker nodes if it doesn't exist +func ensureDropInDirectoryExists(ctx context.Context, oc *exutil.CLI, dirPath string) error { + nodes, err := getNodesByLabel(ctx, oc, "node-role.kubernetes.io/worker") + if err != nil { + return fmt.Errorf("failed to get worker nodes: %w", err) + } + + for _, node := range nodes { + _, err := ExecOnNodeWithChroot(ctx, oc, node.Name, "mkdir", "-p", dirPath) + if err != nil { + framework.Logf("Warning: failed to create directory on node %s: %v", node.Name, err) + } + } + + return nil +} + +// GetFirstReadyWorkerNode returns the name of the first Ready worker node in the cluster. +func GetFirstReadyWorkerNode(oc *exutil.CLI) string { + ctx := context.Background() + nodes, err := oc.AdminKubeClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{ + LabelSelector: "node-role.kubernetes.io/worker", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(nodes.Items).NotTo(o.BeEmpty(), "no worker nodes found") + + for _, node := range nodes.Items { + if role, found := NodeHasCustomRole(node); found { + framework.Logf("Skipping node %s (already has custom role %q)", node.Name, role) + continue + } + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { + return node.Name + } + } + } + o.Expect(false).To(o.BeTrue(), "no Ready worker node without custom roles found") + return "" +} + +// CalculateEventTimeDiff calculates the time difference between two Kubernetes events. +// It uses LastTimestamp for both events to handle repeated events correctly. +// For repeated events (like container restarts), Kubernetes reuses the event object and updates +// LastTimestamp while keeping FirstTimestamp at the original occurrence. +// Falls back to FirstTimestamp if LastTimestamp is zero. +func CalculateEventTimeDiff(startEvent, endEvent *corev1.Event) time.Duration { + startTime := startEvent.LastTimestamp.Time + if startTime.IsZero() { + startTime = startEvent.FirstTimestamp.Time + } + endTime := endEvent.LastTimestamp.Time + if endTime.IsZero() { + endTime = endEvent.FirstTimestamp.Time + } + return endTime.Sub(startTime) +} + +// GetPodNetNs retrieves the network namespace path for a pod using crictl. +// It uses crictl to get the sandbox ID and then inspects it to extract the NetNS path. +// Returns the NetNS path and an error if not found. +func GetPodNetNs(ctx context.Context, oc *exutil.CLI, nodeName, podName string) (string, error) { + // Get sandbox ID using crictl + sandboxID, err := ExecOnNodeWithChroot(ctx, oc, nodeName, "crictl", "pods", "--name", podName, "-q") + if err != nil || sandboxID == "" { + framework.Logf("Failed to get sandbox ID for pod %s: %v", podName, err) + return "", fmt.Errorf("failed to get sandbox ID for pod %s: %w", podName, err) + } + sandboxID = strings.TrimSpace(sandboxID) + framework.Logf("Found sandbox ID: %s", sandboxID) + + // Extract network namespace path from sandbox inspection + netNsStr, err := ExecOnNodeWithChroot(ctx, oc, nodeName, "sh", "-c", fmt.Sprintf("crictl inspectp %s | grep -i netns", sandboxID)) + if err != nil { + framework.Logf("Failed to get NetNS from crictl inspect: %v", err) + return "", fmt.Errorf("failed to get NetNS from crictl inspect: %w", err) + } + + // Extract NetNS path from the output (format: "linux": { "namespaces": [ { "type": "network", "path": "/var/run/netns/..." } ] } ) + re := regexp.MustCompile(`"path":\s*"([^"]+)"`) + matches := re.FindStringSubmatch(netNsStr) + if len(matches) < 2 { + framework.Logf("NetNS path not found in crictl output for pod %s", podName) + return "", fmt.Errorf("NetNS path not found in crictl output for pod %s", podName) + } + netNsPath := matches[1] + framework.Logf("Extracted NetNS path: %v", netNsPath) + return netNsPath, nil +} + +// CheckNetNsCleaned verifies that the network namespace file has been cleaned up. +// It checks if the NetNS path no longer exists on the node. +// Returns nil if the file is cleaned, error if it still exists. +func CheckNetNsCleaned(ctx context.Context, oc *exutil.CLI, nodeName, netNsPath string) error { + // Use test command which returns proper exit code + _, err := ExecOnNodeWithChroot(ctx, oc, nodeName, "test", "-e", netNsPath) + if err != nil { + // Non-nil err: file absent (test exit 1) OR exec/debug failure. + framework.Logf("NetNS file considered cleaned (test -e returned error: %v)", err) + return nil + } + // No error means file still exists + return fmt.Errorf("NetNS file still exists at %s", netNsPath) +} + +func GetNotReadyNodes(ctx context.Context, oc *exutil.CLI) ([]string, error) { + nodes, err := oc.AdminKubeClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + + var notReadyNodes []string + for _, node := range nodes.Items { + if !isNodeInReadyState(&node) { + notReadyNodes = append(notReadyNodes, node.Name) + } + } + + return notReadyNodes, nil +} + +func EnsureNodesReady(ctx context.Context, oc *exutil.CLI) { + notReadyNodes, err := GetNotReadyNodes(ctx, oc) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to check node readiness") + o.Expect(notReadyNodes).To(o.BeEmpty(), + "Cannot start test: nodes not Ready: %v. Cluster may be recovering from previous test.", notReadyNodes) +} diff --git a/test/extended/testdata/bindata.go b/test/extended/testdata/bindata.go index fb54012d613b..e4e9df70f8ed 100644 --- a/test/extended/testdata/bindata.go +++ b/test/extended/testdata/bindata.go @@ -449,6 +449,7 @@ // test/extended/testdata/net-attach-defs/whereabouts-nad.yml // test/extended/testdata/net-attach-defs/whereabouts-race-awake.yml // test/extended/testdata/net-attach-defs/whereabouts-race-sleepy.yml +// test/extended/testdata/node/node_e2e/pod-dev-fuse.yaml // test/extended/testdata/node_tuning/nto-stalld.yaml // test/extended/testdata/oauthserver/cabundle-cm.yaml // test/extended/testdata/oauthserver/oauth-network.yaml @@ -50259,6 +50260,43 @@ func testExtendedTestdataNetAttachDefsWhereaboutsRaceSleepyYml() (*asset, error) return a, nil } +var _testExtendedTestdataNodeNode_e2ePodDevFuseYaml = []byte(`apiVersion: v1 +kind: Pod +metadata: + name: pod-devfuse + annotations: + io.kubernetes.cri-o.Devices: "/dev/fuse" +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: pod-devfuse + image: image-registry.openshift-image-registry.svc:5000/openshift/cli:latest + command: ["sleep", "infinity"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL +`) + +func testExtendedTestdataNodeNode_e2ePodDevFuseYamlBytes() ([]byte, error) { + return _testExtendedTestdataNodeNode_e2ePodDevFuseYaml, nil +} + +func testExtendedTestdataNodeNode_e2ePodDevFuseYaml() (*asset, error) { + bytes, err := testExtendedTestdataNodeNode_e2ePodDevFuseYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/node/node_e2e/pod-dev-fuse.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _testExtendedTestdataNode_tuningNtoStalldYaml = []byte(`apiVersion: tuned.openshift.io/v1 kind: Tuned metadata: @@ -58745,6 +58783,7 @@ var _bindata = map[string]func() (*asset, error){ "test/extended/testdata/net-attach-defs/whereabouts-nad.yml": testExtendedTestdataNetAttachDefsWhereaboutsNadYml, "test/extended/testdata/net-attach-defs/whereabouts-race-awake.yml": testExtendedTestdataNetAttachDefsWhereaboutsRaceAwakeYml, "test/extended/testdata/net-attach-defs/whereabouts-race-sleepy.yml": testExtendedTestdataNetAttachDefsWhereaboutsRaceSleepyYml, + "test/extended/testdata/node/node_e2e/pod-dev-fuse.yaml": testExtendedTestdataNodeNode_e2ePodDevFuseYaml, "test/extended/testdata/node_tuning/nto-stalld.yaml": testExtendedTestdataNode_tuningNtoStalldYaml, "test/extended/testdata/oauthserver/cabundle-cm.yaml": testExtendedTestdataOauthserverCabundleCmYaml, "test/extended/testdata/oauthserver/oauth-network.yaml": testExtendedTestdataOauthserverOauthNetworkYaml, @@ -59535,6 +59574,11 @@ var _bintree = &bintree{nil, map[string]*bintree{ "whereabouts-race-awake.yml": {testExtendedTestdataNetAttachDefsWhereaboutsRaceAwakeYml, map[string]*bintree{}}, "whereabouts-race-sleepy.yml": {testExtendedTestdataNetAttachDefsWhereaboutsRaceSleepyYml, map[string]*bintree{}}, }}, + "node": {nil, map[string]*bintree{ + "node_e2e": {nil, map[string]*bintree{ + "pod-dev-fuse.yaml": {testExtendedTestdataNodeNode_e2ePodDevFuseYaml, map[string]*bintree{}}, + }}, + }}, "node_tuning": {nil, map[string]*bintree{ "nto-stalld.yaml": {testExtendedTestdataNode_tuningNtoStalldYaml, map[string]*bintree{}}, }}, diff --git a/test/extended/testdata/node/node_e2e/pod-dev-fuse.yaml b/test/extended/testdata/node/node_e2e/pod-dev-fuse.yaml new file mode 100644 index 000000000000..69e1f5b6b3f8 --- /dev/null +++ b/test/extended/testdata/node/node_e2e/pod-dev-fuse.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod-devfuse + annotations: + io.kubernetes.cri-o.Devices: "/dev/fuse" +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: pod-devfuse + image: image-registry.openshift-image-registry.svc:5000/openshift/cli:latest + command: ["sleep", "infinity"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL diff --git a/test/extended/util/annotate/generated/zz_generated.annotations.go b/test/extended/util/annotate/generated/zz_generated.annotations.go index dd7b9e963a20..8b566226e5da 100644 --- a/test/extended/util/annotate/generated/zz_generated.annotations.go +++ b/test/extended/util/annotate/generated/zz_generated.annotations.go @@ -31,6 +31,14 @@ var Annotations = map[string]string{ "[Serial] [sig-auth][Feature:OAuthServer] [RequestHeaders] [IdP] test RequestHeaders IdP [apigroup:config.openshift.io][apigroup:user.openshift.io]": " [Suite:openshift/conformance/serial]", + "[Suite:openshift/disruptive-longrunning][sig-node][Disruptive] ContainerRuntimeConfig [OTP] Verify overlaySize is applied to node and container [OCP-46313]": " [Serial]", + + "[Suite:openshift/disruptive-longrunning][sig-node][Disruptive] ContainerRuntimeConfig [OTP] Verify pidsLimit and MCO overwrite behavior [OCP-45351]": " [Serial]", + + "[Suite:openshift/disruptive-longrunning][sig-node][Disruptive] Image registry config [OTP] change container registry config [OCP-44820]": " [Serial]", + + "[Suite:openshift/disruptive-longrunning][sig-node][Disruptive] PodDisruptionBudget [OTP] Node's drain should block when PodDisruptionBudget minAvailable equals 100 percentage and selector is empty [OCP-67564]": " [Serial]", + "[Suite:openshift/machine-config-operator/disruptive][Suite:openshift/conformance/serial][sig-mco][OCPFeatureGate:ManagedBootImagesAWS][Serial] Should degrade on a MachineSet with an OwnerReference [apigroup:machineconfiguration.openshift.io]": "", "[Suite:openshift/machine-config-operator/disruptive][Suite:openshift/conformance/serial][sig-mco][OCPFeatureGate:ManagedBootImagesAWS][Serial] Should not update boot images on any MachineSet when not configured [apigroup:machineconfiguration.openshift.io]": "", @@ -1843,8 +1851,24 @@ var Annotations = map[string]string{ "[sig-node] Managed cluster should verify that nodes have no unexpected reboots [Late]": " [Suite:openshift/conformance/parallel]", + "[sig-node] Probe configuration [OTP] Liveness probe should fall back to pod-level terminationGracePeriodSeconds when probe-level is not set [OCP-44493]": " [Suite:openshift/conformance/parallel]", + + "[sig-node] Probe configuration [OTP] Liveness probe should respect probe-level terminationGracePeriodSeconds [OCP-44493]": " [Suite:openshift/conformance/parallel]", + + "[sig-node] Probe configuration [OTP] Startup probe should respect probe-level terminationGracePeriodSeconds [OCP-44493]": " [Suite:openshift/conformance/parallel]", + "[sig-node] [Conformance] Prevent openshift node labeling on update by the node TestOpenshiftNodeLabeling": " [Suite:openshift/conformance/parallel/minimal]", + "[sig-node] [Jira:Node/Kubelet] Kubelet, CRI-O, CPU manager [OTP] Allow dev fuse by default in CRI-O [OCP-70987]": " [Suite:openshift/conformance/parallel]", + + "[sig-node] [Jira:Node/Kubelet] Kubelet, CRI-O, CPU manager [OTP] validate KUBELET_LOG_LEVEL": " [Suite:openshift/conformance/parallel]", + + "[sig-node] [Jira:Node/Kubelet] Kubelet, CRI-O, CPU manager [OTP] validate cgroupv2 is default [OCP-80983]": " [Suite:openshift/conformance/parallel]", + + "[sig-node] [Jira:Node/Kubelet] NODE initContainer policy,volume,readiness,quota [OTP] Init containers should not restart when the exited init container is removed from node [OCP-38271]": " [Suite:openshift/conformance/parallel]", + + "[sig-node] [Jira:Node/Kubelet] Network namespace cleanup [OTP] kubelet/crio will delete netns when a pod is deleted [OCP-56266]": " [Suite:openshift/conformance/parallel]", + "[sig-node] should override timeoutGracePeriodSeconds when annotation is set": " [Suite:openshift/conformance/parallel]", "[sig-node] supplemental groups Ensure supplemental groups propagate to docker should propagate requested groups to the container [apigroup:security.openshift.io]": " [Suite:openshift/conformance/parallel]", @@ -1853,6 +1877,10 @@ var Annotations = map[string]string{ "[sig-node][Late] should not have pod creation failures due to systemd timeouts": " [Suite:openshift/conformance/parallel]", + "[sig-node][Suite:openshift/disruptive-longrunning][Disruptive][Serial] ImageTagMirrorSet and ImageDigestMirrorSet [OTP] Create ImageDigestMirrorSet and ImageTagMirrorSet and verify registries.conf [OCP-57401]": "", + + "[sig-node][Suite:openshift/disruptive-longrunning][Disruptive][Serial] ImageTagMirrorSet and ImageDigestMirrorSet [OTP] ICSP and IDMS/ITMS can coexist in cluster [OCP-70203]": "", + "[sig-node][Suite:openshift/nodes/realtime/latency][Disruptive] Real time kernel should meet latency requirements when tested with cyclictest": " [Serial]", "[sig-node][Suite:openshift/nodes/realtime/latency][Disruptive] Real time kernel should meet latency requirements when tested with hwlatdetect": " [Serial]",