From 9c6ed459a651ca4f061b29ec41be903c63e0eb17 Mon Sep 17 00:00:00 2001 From: ehila Date: Tue, 11 Aug 2026 12:51:20 +0300 Subject: [PATCH 1/2] Allow DualReplica stable-system CVO Available blips from NoExecuteTaintManager Serial NoExecuteTaintManager tests taint a control-plane node on DualReplica (masters are also workers), briefly driving csi-snapshot-controller and packageserver Available=False. Mirror the existing upgrade-path exceptions in the stable-system monitor so TNF serial jobs do not fail on these expected blips. Co-authored-by: Cursor Signed-off-by: ehila --- .../legacycvomonitortests/operators.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go index 3af608055c79..7e8a408afcf9 100644 --- a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go +++ b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go @@ -48,6 +48,7 @@ func checkAuthenticationAvailableExceptions(condition *configv1.ClusterOperatorS } func testStableSystemOperatorStateTransitions(events monitorapi.Intervals, topology configv1.TopologyMode) []*junitapi.JUnitTestCase { + isTwoNodeDualReplica := topology == configv1.DualReplicaTopologyMode except := func(operator string, condition *configv1.ClusterOperatorStatusCondition, _ monitorapi.Interval) string { if condition.Status == configv1.ConditionTrue { if condition.Type == configv1.OperatorAvailable { @@ -86,6 +87,20 @@ func testStableSystemOperatorStateTransitions(events monitorapi.Intervals, topol if operator == "ingress" && condition.Reason == "IngressUnavailable" { return "https://issues.redhat.com/browse/OCPBUGS-92835" } + // DualReplica / TNF: serial NoExecuteTaintManager tests NoExecute-taint a control-plane + // node (masters are also workers), so Deployments briefly cannot schedule. + if isTwoNodeDualReplica { + switch operator { + case "csi-snapshot-controller": + if strings.Contains(condition.Message, `Waiting for Deployment`) { + return "csi-snapshot-controller may report Available=False while Waiting for Deployment during DualReplica disruptive serial tests (e.g. NoExecuteTaintManager)" + } + case "operator-lifecycle-manager-packageserver": + if condition.Reason == "ClusterServiceVersionNotSucceeded" { + return "https://issues.redhat.com/browse/OCPBUGS-23744" + } + } + } return "" } if condition.Type == configv1.OperatorDegraded && condition.Status == configv1.ConditionTrue { From 638822f18f565970bc6609220ba485789420db77 Mon Sep 17 00:00:00 2001 From: ehila Date: Thu, 13 Aug 2026 09:44:31 +0200 Subject: [PATCH 2/2] Require NoExecuteTaintManager overlap for DualReplica Available exceptions Scope the stable-system csi-snapshot-controller and packageserver Available allowlist to blips that overlap a NoExecuteTaintManager e2e window, plus a one-minute grace for post-taint recovery, instead of any DualReplica run. Do not key off a serial job signal: serial only describes how NoExecuteTaintManager enters the suite, not that it caused a given Available blip. Overlap with the test interval is the causal check and still fails unrelated DualReplica Available=False regressions outside that window. Co-authored-by: Cursor Signed-off-by: ehila --- .../legacycvomonitortests/operators.go | 38 +++++++++- .../legacycvomonitortests/operators_test.go | 74 +++++++++++++++++++ 2 files changed, 108 insertions(+), 4 deletions(-) diff --git a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go index 7e8a408afcf9..0946da9456d7 100644 --- a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go +++ b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go @@ -47,9 +47,37 @@ func checkAuthenticationAvailableExceptions(condition *configv1.ClusterOperatorS return false } +// noExecuteTaintManagerAvailableGrace covers brief Available blips that land just after +// NoExecuteTaintManager removes its taint while Deployments are still rescheduling +// (observed packageserver recovery ~3s after test end). +const noExecuteTaintManagerAvailableGrace = time.Minute + +func noExecuteTaintManagerTestIntervals(events monitorapi.Intervals) monitorapi.Intervals { + return events.Filter(func(eventInterval monitorapi.Interval) bool { + return eventInterval.Source == monitorapi.SourceE2ETest && + strings.Contains(eventInterval.Locator.Keys[monitorapi.LocatorE2ETestKey], "NoExecuteTaintManager") + }) +} + +// overlapsNoExecuteTaintManagerTest reports whether conditionInterval overlaps a +// NoExecuteTaintManager e2e window, including grace after the test ends. +func overlapsNoExecuteTaintManagerTest(conditionInterval monitorapi.Interval, taintTests monitorapi.Intervals, grace time.Duration) bool { + for _, test := range taintTests { + window := test + if !window.To.IsZero() { + window.To = window.To.Add(grace) + } + if utility.IntervalsOverlap(conditionInterval, window) { + return true + } + } + return false +} + func testStableSystemOperatorStateTransitions(events monitorapi.Intervals, topology configv1.TopologyMode) []*junitapi.JUnitTestCase { isTwoNodeDualReplica := topology == configv1.DualReplicaTopologyMode - except := func(operator string, condition *configv1.ClusterOperatorStatusCondition, _ monitorapi.Interval) string { + taintTests := noExecuteTaintManagerTestIntervals(events) + except := func(operator string, condition *configv1.ClusterOperatorStatusCondition, eventInterval monitorapi.Interval) string { if condition.Status == configv1.ConditionTrue { if condition.Type == configv1.OperatorAvailable { return fmt.Sprintf("%s=%s is the happy case", condition.Type, condition.Status) @@ -88,12 +116,14 @@ func testStableSystemOperatorStateTransitions(events monitorapi.Intervals, topol return "https://issues.redhat.com/browse/OCPBUGS-92835" } // DualReplica / TNF: serial NoExecuteTaintManager tests NoExecute-taint a control-plane - // node (masters are also workers), so Deployments briefly cannot schedule. - if isTwoNodeDualReplica { + // node (masters are also workers), so Deployments briefly cannot schedule. Require the + // blip to overlap a NoExecuteTaintManager window (+grace) so unrelated Available=False + // regressions are not allowlisted for the whole DualReplica job. + if isTwoNodeDualReplica && overlapsNoExecuteTaintManagerTest(eventInterval, taintTests, noExecuteTaintManagerAvailableGrace) { switch operator { case "csi-snapshot-controller": if strings.Contains(condition.Message, `Waiting for Deployment`) { - return "csi-snapshot-controller may report Available=False while Waiting for Deployment during DualReplica disruptive serial tests (e.g. NoExecuteTaintManager)" + return "csi-snapshot-controller may report Available=False while Waiting for Deployment during DualReplica NoExecuteTaintManager tests" } case "operator-lifecycle-manager-packageserver": if condition.Reason == "ClusterServiceVersionNotSucceeded" { diff --git a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators_test.go b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators_test.go index ecdb11142232..0c135acd1232 100644 --- a/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators_test.go +++ b/pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators_test.go @@ -490,3 +490,77 @@ func TestIsTNFJobClusterOperatorReason(t *testing.T) { }) } } + +func TestOverlapsNoExecuteTaintManagerTest(t *testing.T) { + base := time.Date(2026, 8, 10, 13, 55, 43, 0, time.UTC) + taintTest := monitorapi.Interval{ + Condition: monitorapi.Condition{ + Locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorE2ETestKey: "[sig-node] NoExecuteTaintManager Single Pod [Serial] eventually evict pod with finite tolerations from tainted nodes", + }, + }, + }, + Source: monitorapi.SourceE2ETest, + From: base, + To: base.Add(2 * time.Minute), // ends 13:57:55 + } + otherTest := monitorapi.Interval{ + Condition: monitorapi.Condition{ + Locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorE2ETestKey: "[sig-api-machinery] Namespaces [Serial] should ensure that all pods are removed when a namespace is deleted", + }, + }, + }, + Source: monitorapi.SourceE2ETest, + From: base.Add(2 * time.Minute), + To: base.Add(3 * time.Minute), + } + taintTests := noExecuteTaintManagerTestIntervals(monitorapi.Intervals{taintTest, otherTest}) + assert.Len(t, taintTests, 1) + + tests := []struct { + name string + from time.Time + to time.Time + want bool + }{ + { + name: "during test", + from: base.Add(30 * time.Second), + to: base.Add(45 * time.Second), + want: true, + }, + { + name: "few seconds after test within grace", + from: base.Add(2*time.Minute + 3*time.Second), + to: base.Add(2*time.Minute + 3*time.Second + 81*time.Millisecond), + want: true, + }, + { + name: "just inside grace window", + from: base.Add(2*time.Minute + 59*time.Second), + to: base.Add(3 * time.Minute), + want: true, + }, + { + name: "after grace window", + from: base.Add(3*time.Minute + time.Second), + to: base.Add(3*time.Minute + 2*time.Second), + want: false, + }, + { + name: "before test", + from: base.Add(-2 * time.Minute), + to: base.Add(-time.Minute), + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + condition := monitorapi.Interval{From: tt.from, To: tt.to} + assert.Equal(t, tt.want, overlapsNoExecuteTaintManagerTest(condition, taintTests, noExecuteTaintManagerAvailableGrace)) + }) + } +}