[release-4.19] OCPBUGS-104577: Backport node_e2e test migrations

go.mod

@@ -77,6 +77,7 @@ require (
 	k8s.io/kube-aggregator v0.32.8
 	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f
 	k8s.io/kubectl v0.32.8
+	k8s.io/kubelet v0.31.1
 	k8s.io/kubernetes v1.32.8
 	k8s.io/pod-security-admission v0.32.8
 	k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d
@@ -303,7 +304,6 @@ require (
 	k8s.io/externaljwt v0.0.0 // indirect
 	k8s.io/kms v0.32.1 // indirect
 	k8s.io/kube-scheduler v0.0.0 // indirect
-	k8s.io/kubelet v0.31.1 // indirect
 	k8s.io/mount-utils v0.0.0 // indirect
 	k8s.io/sample-apiserver v0.0.0 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect

test/extended/imagepolicy/imagepolicy_helpers.go

@@ -0,0 +1,41 @@
+package imagepolicy
+
+import (
+	"context"
+	"time"
+
+	o "github.com/onsi/gomega"
+	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
+	machineconfigclient "github.com/openshift/client-go/machineconfiguration/clientset/versioned"
+	machineconfighelper "github.com/openshift/origin/test/extended/machine_config"
+	exutil "github.com/openshift/origin/test/extended/util"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
+)
+
+// GetMCPCurrentSpecConfigName returns the current Spec.Configuration.Name for the given MachineConfigPool.
+func GetMCPCurrentSpecConfigName(oc *exutil.CLI, pool string) string {
+	clientSet, err := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
+	o.Expect(err).NotTo(o.HaveOccurred())
+	mcp, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(context.TODO(), pool, metav1.GetOptions{})
+	o.Expect(err).NotTo(o.HaveOccurred())
+	return mcp.Spec.Configuration.Name
+}
+
+// WaitForMCPConfigSpecChangeAndUpdated waits until Spec.Configuration.Name changes from the provided initial value
+// and the MCP reports Updated=true.
+func WaitForMCPConfigSpecChangeAndUpdated(oc *exutil.CLI, pool string, initialSpecName string) {
+	e2e.Logf("Waiting for pool %s to complete", pool)
+	clientSet, err := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
+	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Eventually(func() bool {
+		mcp, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(context.TODO(), pool, metav1.GetOptions{})
+		if err != nil {
+			return false
+		}
+		if mcp.Status.Configuration.Name == initialSpecName {
+			return false
+		}
+		return machineconfighelper.IsMachineConfigPoolConditionTrue(mcp.Status.Conditions, mcfgv1.MachineConfigPoolUpdated)
+	}, 20*time.Minute, 10*time.Second).Should(o.BeTrue())
+}

test/extended/include.go

@@ -40,6 +40,7 @@ import (
 	_ "github.com/openshift/origin/test/extended/machine_config"
 	_ "github.com/openshift/origin/test/extended/machines"
 	_ "github.com/openshift/origin/test/extended/networking"
+	_ "github.com/openshift/origin/test/extended/node/node_e2e"
 	_ "github.com/openshift/origin/test/extended/node_tuning"
 	_ "github.com/openshift/origin/test/extended/oauth"
 	_ "github.com/openshift/origin/test/extended/olm"

test/extended/node/README.md

@@ -0,0 +1,77 @@
+# Node E2E Tests
+
+This directory contains OpenShift end-to-end tests for node-related features.
+
+## Test Suites
+
+### Suite: openshift/disruptive-longrunning
+
+- **node_e2e/container_runtime_config.go** - ContainerRuntimeConfig pidsLimit (OCP-45351) and overlaySize (OCP-46313) - Verifies CTRCFG settings are applied via MCO rollout and reflected on nodes \[Disruptive\]
+- **node_e2e/image_mirror_set.go** - ImageDigestMirrorSet and ImageTagMirrorSet (OCP-57401, OCP-70203) - Verifies registries.conf reflects IDMS/ITMS configuration and that ICSP/IDMS/ITMS can coexist \[Disruptive\] \[Serial\]
+- **node_e2e/image_registry_config.go** - Container registry config change (OCP-44820) - Verifies search registry update triggers MCO rollout and lands on nodes \[Disruptive\]
+- **node_e2e/pdb_drain.go** - PodDisruptionBudget drain blocking (OCP-67564) - Tests that node drain is blocked when PDB has minAvailable=100% with empty selector \[Disruptive\]
+
+### Suite: openshift/conformance/parallel
+
+- **node_e2e/initcontainer.go** - Init container restart behavior (OCP-38271) - Verifies init containers do not restart when the exited init container is removed from the node
+- **node_e2e/netns_cleanup.go** - Network namespace cleanup (OCP-56266) - Verifies kubelet/CRI-O properly deletes the network namespace when a pod is deleted
+- **node_e2e/node.go** - Kubelet log level (KUBELET_LOG_LEVEL), cgroupv2 default validation, and dev fuse enablement in CRI-O (OCP-80983, OCP-70987)
+- **node_e2e/probe_termination.go** - Probe-level terminationGracePeriodSeconds (OCP-44493) - Tests configurable termination grace period for liveness and startup probes, including fallback to pod-level config when probe-level is not set
+
+## Directory Structure
+
+### Test Files
+- All `*.go` files under `node_e2e/` are Ginkgo-based test suites
+- Each file focuses on a specific node feature
+
+### Utility Files
+- **node_utils.go** - Shared helper functions for node selection, exec-on-node, and MachineConfigPool rollout waiting
+- **node_mcp_helpers.go** - Custom MachineConfigPool creation/cleanup helpers
+- **../imagepolicy/imagepolicy_helpers.go** - MachineConfigPool spec-name helpers shared with `node_e2e` tests
+
+### Test Data
+Test fixtures are referenced via `exutil.FixturePath` from:
+- `testdata/node/node_e2e/` - Pod fixtures (e.g. dev fuse test pod)
+
+## Running Tests
+
+### Running Long-Running Disruptive Tests
+
+The `openshift/disruptive-longrunning` suite is a general-purpose suite for long-running disruptive tests
+across all teams. Node team tests are tagged with `[sig-node]` to identify them.
+
+To run the entire long-running disruptive test suite on a cluster manually:
+
+```bash
+./openshift-tests run "openshift/disruptive-longrunning" --cluster-stability=Disruptive
+```
+
+To run only node-specific long-running disruptive tests:
+
+```bash
+./openshift-tests run "openshift/disruptive-longrunning" --dry-run | grep "\[sig-node\]" | ./openshift-tests run -f - --cluster-stability=Disruptive
+```
+
+## Prerequisites
+
+- Make sure to set `oc` binary to match the cluster version
+- Make sure to set the kubeconfig to point to a live OCP cluster
+
+## Submitting PRs
+
+### Adding Tests to `openshift/disruptive-longrunning`
+
+Before submitting a PR that adds a test to the `openshift/disruptive-longrunning` suite, run the following payload job and include the results in your PR:
+
+```
+/payload-job periodic-ci-openshift-release-main-nightly-4.19-e2e-aws-disruptive-longrunning
+```
+
+Useful links for `periodic-ci-openshift-release-main-nightly-4.19-e2e-aws-disruptive-longrunning`:
+- [Previous runs (Sippy)](https://sippy.dptools.openshift.org/sippy-ng/jobs/4.19/analysis?filters=%7B%22items%22%3A%5B%7B%22columnField%22%3A%22name%22%2C%22operatorValue%22%3A%22equals%22%2C%22value%22%3A%22periodic-ci-openshift-release-main-nightly-4.19-e2e-aws-disruptive-longrunning%22%7D%5D%7D)
+- [Job history for latest runs (Prow)](https://prow.ci.openshift.org/job-history/gs/test-platform-results/logs/periodic-ci-openshift-release-main-nightly-4.19-e2e-aws-disruptive-longrunning)
+
+## Important Notes
+
+- Note that dry-run option won't list the test as it does not connect to a live cluster
+- Run `make update` if the test data is changed

test/extended/node/node_e2e/OWNERS

@@ -0,0 +1,10 @@
+reviewers:
+  - asahay19
+  - cpmeadors
+  - sairameshv
+  - mrunalp
+  - BhargaviGudi
+approvers:
+  - cpmeadors
+  - sairameshv
+  - mrunalp

test/extended/node/node_e2e/container_runtime_config.go

@@ -0,0 +1,215 @@
+package node
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	g "github.com/onsi/ginkgo/v2"
+	o "github.com/onsi/gomega"
+
+	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
+	machineconfigclient "github.com/openshift/client-go/machineconfiguration/clientset/versioned"
+	"github.com/openshift/origin/test/extended/imagepolicy"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/utils/ptr"
+
+	nodeutils "github.com/openshift/origin/test/extended/node"
+	exutil "github.com/openshift/origin/test/extended/util"
+)
+
+var _ = g.Describe("[Suite:openshift/disruptive-longrunning][sig-node][Disruptive] ContainerRuntimeConfig", func() {
+	var (
+		oc = exutil.NewCLIWithoutNamespace("ctrcfg")
+	)
+
+	g.BeforeEach(func(ctx context.Context) {
+		nodeutils.SkipOnMicroShift(oc)
+		nodeutils.EnsureNodesReady(ctx, oc)
+	})
+
+	// Validates that ContainerRuntimeConfig pidsLimit setting is correctly applied
+	// by MCO to a single worker node and that manual crio.conf edits are overwritten.
+	//author: cmaurya@redhat.com
+	g.It("[OTP] Verify pidsLimit and MCO overwrite behavior [OCP-45351]", func() {
+		ctx := context.Background()
+		ctrcfgName := "set-pids-limit"
+		mcpName := "ctrcfg-pids"
+
+		g.By("Get a ready worker node")
+		workerNode := nodeutils.GetFirstReadyWorkerNode(oc)
+		o.Expect(workerNode).NotTo(o.BeEmpty(), "no ready worker node found")
+		err := nodeutils.EnsureNodeHasNoCustomRole(ctx, oc, workerNode)
+		o.Expect(err).NotTo(o.HaveOccurred())
+
+		g.By("Make a manual change to crio.conf on worker node")
+		_, err = nodeutils.ExecOnNodeWithChroot(ctx, oc, workerNode,
+			"/bin/bash", "-c", `sed -i '/^\[crio\.runtime\]/a log_level = "debug"' /etc/crio/crio.conf`)
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to edit crio.conf on node %s", workerNode)
+
+		g.By("Verify the manual crio.conf edit took effect")
+		editedConf, err := nodeutils.ExecOnNodeWithChroot(ctx, oc, workerNode, "cat", "/etc/crio/crio.conf")
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to read crio.conf on node %s", workerNode)
+		o.Expect(editedConf).To(o.ContainSubstring(`log_level = "debug"`),
+			"sed edit did not apply: expected log_level = debug in crio.conf")
+
+		mcClient, err := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to create machine config client")
+
+		var mcpConfig *nodeutils.CustomMCPConfig
+		g.DeferCleanup(func() {
+			cleanupCtx := context.Background()
+			delErr := oc.MachineConfigurationClient().MachineconfigurationV1().ContainerRuntimeConfigs().Delete(
+				cleanupCtx, ctrcfgName, metav1.DeleteOptions{})
+			if delErr != nil && !apierrors.IsNotFound(delErr) {
+				e2e.Logf("Warning: failed to delete ContainerRuntimeConfig %s: %v", ctrcfgName, delErr)
+			}
+			if err := nodeutils.CleanupCustomMCP(cleanupCtx, mcpConfig); err != nil {
+				e2e.Logf("WARNING: cleanup had errors: %v", err)
+			}
+		})
+
+		mcpConfig, err = nodeutils.CreateCustomMCPForNode(ctx, oc, mcClient, mcpName, workerNode)
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to create custom MCP")
+
+		initialSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, mcpName)
+
+		g.By("Create ContainerRuntimeConfig with pidsLimit 2048")
+		ctrcfg := &mcfgv1.ContainerRuntimeConfig{
+			ObjectMeta: metav1.ObjectMeta{Name: ctrcfgName},
+			Spec: mcfgv1.ContainerRuntimeConfigSpec{
+				MachineConfigPoolSelector: &metav1.LabelSelector{
+					MatchLabels: map[string]string{"machineconfiguration.openshift.io/pool": mcpName},
+				},
+				ContainerRuntimeConfig: &mcfgv1.ContainerRuntimeConfiguration{
+					PidsLimit: ptr.To[int64](2048),
+				},
+			},
+		}
+		_, err = oc.MachineConfigurationClient().MachineconfigurationV1().ContainerRuntimeConfigs().Create(
+			ctx, ctrcfg, metav1.CreateOptions{})
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to create ContainerRuntimeConfig")
+
+		g.By("Wait for custom MCP rollout to complete")
+		imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, mcpName, initialSpec)
+		e2e.Logf("Worker node rolled out successfully")
+
+		g.By("Verify pidsLimit and conmon in crio config on worker node")
+		var crioConfig string
+		o.Eventually(func() error {
+			var execErr error
+			crioConfig, execErr = nodeutils.ExecOnNodeWithChroot(ctx, oc, workerNode,
+				"/bin/bash", "-c", "crio config 2>/dev/null")
+			return execErr
+		}, 30*time.Second, 5*time.Second).Should(o.Succeed(), "failed to get crio config on node %s", workerNode)
+		o.Expect(crioConfig).To(o.ContainSubstring("pids_limit = 2048"), "pidsLimit should be 2048")
+		o.Expect(crioConfig).To(o.ContainSubstring(`conmon = ""`), "conmon should be empty")
+		o.Expect(crioConfig).NotTo(o.ContainSubstring(`log_level = "debug"`),
+			"manual crio.conf edit should be overwritten by MCO")
+	})
+
+	// Validates that setting overlaySize in ContainerRuntimeConfig is applied to
+	// storage.conf on a single worker node and the overlay size is reflected inside a container.
+	//author: cmaurya@redhat.com
+	g.It("[OTP] Verify overlaySize is applied to node and container [OCP-46313]", func() {
+		oc.SetupProject()
+		ctx := context.Background()
+		ctrcfgName := "ctrcfg-46313"
+		mcpName := "ctrcfg-overlay"
+		overlaySize := "9G"
+
+		g.By("Get a ready worker node")
+		workerNode := nodeutils.GetFirstReadyWorkerNode(oc)
+		o.Expect(workerNode).NotTo(o.BeEmpty(), "no ready worker node found")
+		err := nodeutils.EnsureNodeHasNoCustomRole(ctx, oc, workerNode)
+		o.Expect(err).NotTo(o.HaveOccurred())
+
+		mcClient, err := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to create machine config client")
+
+		var mcpConfig *nodeutils.CustomMCPConfig
+		g.DeferCleanup(func() {
+			cleanupCtx := context.Background()
+			delErr := oc.MachineConfigurationClient().MachineconfigurationV1().ContainerRuntimeConfigs().Delete(
+				cleanupCtx, ctrcfgName, metav1.DeleteOptions{})
+			if delErr != nil && !apierrors.IsNotFound(delErr) {
+				e2e.Logf("Warning: failed to delete ContainerRuntimeConfig %s: %v", ctrcfgName, delErr)
+			}
+			if err := nodeutils.CleanupCustomMCP(cleanupCtx, mcpConfig); err != nil {
+				e2e.Logf("WARNING: cleanup had errors: %v", err)
+			}
+		})
+
+		mcpConfig, err = nodeutils.CreateCustomMCPForNode(ctx, oc, mcClient, mcpName, workerNode)
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to create custom MCP")
+
+		initialSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, mcpName)
+
+		g.By("Create ContainerRuntimeConfig with overlaySize " + overlaySize)
+		quantity := resource.MustParse(overlaySize)
+		ctrcfg := &mcfgv1.ContainerRuntimeConfig{
+			ObjectMeta: metav1.ObjectMeta{Name: ctrcfgName},
+			Spec: mcfgv1.ContainerRuntimeConfigSpec{
+				MachineConfigPoolSelector: &metav1.LabelSelector{
+					MatchLabels: map[string]string{"machineconfiguration.openshift.io/pool": mcpName},
+				},
+				ContainerRuntimeConfig: &mcfgv1.ContainerRuntimeConfiguration{
+					OverlaySize: &quantity,
+				},
+			},
+		}
+		_, err = oc.MachineConfigurationClient().MachineconfigurationV1().ContainerRuntimeConfigs().Create(
+			ctx, ctrcfg, metav1.CreateOptions{})
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to create ContainerRuntimeConfig")
+
+		g.By("Wait for custom MCP rollout to complete")
+		imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, mcpName, initialSpec)
+		e2e.Logf("Worker node rolled out successfully")
+
+		g.By("Check overlaySize takes effect in storage.conf on worker node")
+		storageConf, err := nodeutils.ExecOnNodeWithChroot(ctx, oc, workerNode,
+			"/bin/bash", "-c", "head -n 7 /etc/containers/storage.conf | grep size")
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to read storage.conf on node %s", workerNode)
+		e2e.Logf("storage.conf size line: %s", storageConf)
+		o.Expect(storageConf).To(o.ContainSubstring(overlaySize),
+			"storage.conf should contain size = %s", overlaySize)
+
+		g.By("Create a pod on the target node to verify overlay size inside container")
+		podName := "pod-46313"
+		ns := oc.Namespace()
+		err = oc.AsAdmin().WithoutNamespace().Run("run").Args(
+			podName, "-n", ns,
+			"--image=quay.io/openshifttest/hello-openshift@sha256:56c354e7885051b6bb4263f9faa58b2c292d44790599b7dde0e49e7c466cf339",
+			"--restart=Never",
+			"--overrides", `{"spec":{"nodeName":"`+workerNode+`","securityContext":{"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}},"containers":[{"name":"`+podName+`","image":"quay.io/openshifttest/hello-openshift@sha256:56c354e7885051b6bb4263f9faa58b2c292d44790599b7dde0e49e7c466cf339","command":["/bin/bash","-c","sleep 100000000"],"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}}]}}`,
+		).Execute()
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to create pod")
+		defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", podName, "-n", ns, "--ignore-not-found").Execute()
+
+		g.By("Wait for pod to be running")
+		err = wait.Poll(5*time.Second, 5*time.Minute, func() (bool, error) {
+			phase, pollErr := oc.AsAdmin().WithoutNamespace().Run("get").Args(
+				"pod", podName, "-n", ns, "-o=jsonpath={.status.phase}").Output()
+			if pollErr != nil {
+				return false, nil
+			}
+			return phase == "Running", nil
+		})
+		o.Expect(err).NotTo(o.HaveOccurred(), "pod did not reach Running state")
+
+		g.By("Check overlay filesystem size inside the container")
+		dfOutput, err := oc.AsAdmin().WithoutNamespace().Run("rsh").Args(
+			"-n", ns, podName, "/bin/bash", "-c", "df -h / | grep overlay").Output()
+		o.Expect(err).NotTo(o.HaveOccurred(), "failed to exec df inside pod")
+		e2e.Logf("overlay df output: %s", dfOutput)
+		fields := strings.Fields(dfOutput)
+		o.Expect(len(fields)).To(o.BeNumerically(">=", 2), "unexpected df output format: %s", dfOutput)
+		actualSize := strings.Split(strings.TrimSuffix(fields[1], "G"), ".")[0] + "G"
+		o.Expect(actualSize).To(o.Equal(overlaySize),
+			"overlay filesystem should show %s, got: %s", overlaySize, actualSize)
+	})
+})