From 53452f37ce7dadad808fc70ecb4d8dba7cbf428a Mon Sep 17 00:00:00 2001 From: Mikhail Petrov Date: Tue, 30 Jun 2026 23:17:26 +0300 Subject: [PATCH] IGNITE-28837 Added support for multiple independent feature sets during RU --- .../ignite/internal/CoreMessagesProvider.java | 8 +- .../IgniteComponentUpgradeState.java | 93 +++ ...ta.java => RollingUpgradeClusterData.java} | 15 +- .../RollingUpgradeProcessor.java | 316 +++++---- .../feature/IgniteComponentFeatures.java | 118 ++++ .../IgniteComponentFeaturesProvider.java | 42 ++ .../feature/IgniteCoreFeature.java | 8 + .../rollingupgrade/feature/IgniteFeature.java | 4 + .../feature/IgniteFeatureManager.java | 74 +- .../feature/IgniteFeatureSet.java | 8 +- .../feature/IgniteFeatures.java | 111 +++ .../feature/IgniteProductFeatures.java | 80 --- .../AbstractRollingUpgradeTest.java | 648 ++++++++++++++++++ .../ClusterVersionsRollingUpgradeTest.java | 255 +------ .../PluginVersionRollingUpgradeTest.java | 150 ++++ .../feature/AbstractRollingUpgradeTest.java | 265 ------- .../TestPluginComponentFeaturesProvider.java | 54 ++ .../feature/TestPluginFeature.java | 42 ++ .../TestPluginReleaseFeatures_1_0_0.java | 24 + .../TestPluginReleaseFeatures_2_0_0.java | 24 + .../TestPluginReleaseFeatures_3_0_0.java | 24 + .../testsuites/IgniteBasicTestSuite.java | 2 + 22 files changed, 1604 insertions(+), 761 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/IgniteComponentUpgradeState.java rename modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/{RollingUpgradeNodeData.java => RollingUpgradeClusterData.java} (78%) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteComponentFeatures.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteComponentFeaturesProvider.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatures.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteProductFeatures.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PluginVersionRollingUpgradeTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/AbstractRollingUpgradeTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginComponentFeaturesProvider.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginFeature.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginReleaseFeatures_1_0_0.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginReleaseFeatures_2_0_0.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginReleaseFeatures_3_0_0.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java index f40b14826cc1a..b113c2b5de17a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java @@ -236,9 +236,9 @@ import org.apache.ignite.internal.processors.query.stat.messages.StatisticsResponse; import org.apache.ignite.internal.processors.rest.handlers.task.GridTaskResultRequest; import org.apache.ignite.internal.processors.rest.handlers.task.GridTaskResultResponse; -import org.apache.ignite.internal.processors.rollingupgrade.RollingUpgradeNodeData; +import org.apache.ignite.internal.processors.rollingupgrade.RollingUpgradeClusterData; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteComponentFeatures; import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureSet; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteProductFeatures; import org.apache.ignite.internal.processors.service.ServiceChangeBatchRequest; import org.apache.ignite.internal.processors.service.ServiceClusterDeploymentResult; import org.apache.ignite.internal.processors.service.ServiceClusterDeploymentResultBatch; @@ -692,8 +692,8 @@ public CoreMessagesProvider(Marshaller dfltMarsh, Marshaller schemaAwareMarsh, C // [13500 - 13600]: Rolling Upgrade messages. msgIdx = 13500; withNoSchema(IgniteFeatureSet.class); - withNoSchema(IgniteProductFeatures.class); - withNoSchema(RollingUpgradeNodeData.class); + withNoSchema(IgniteComponentFeatures.class); + withNoSchema(RollingUpgradeClusterData.class); assert msgIdx <= MAX_MESSAGE_ID; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/IgniteComponentUpgradeState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/IgniteComponentUpgradeState.java new file mode 100644 index 0000000000000..b915566130c2d --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/IgniteComponentUpgradeState.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rollingupgrade; + +import org.apache.ignite.lang.IgniteProductVersion; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.internal.IgniteVersionUtils.semanticVersion; + +/** */ +class IgniteComponentUpgradeState { + /** */ + final String cmpName; + + /** */ + @Nullable final IgniteProductVersion srcVer; + + /** */ + @Nullable final IgniteProductVersion targetVer; + + /** Whether all cluster nodes hold the same Ignite component version. */ + final boolean isClusterVerHomogenous; + + /** */ + IgniteComponentUpgradeState( + String cmpName, + @Nullable IgniteProductVersion srcVer, + @Nullable IgniteProductVersion targetVer, + boolean isClusterVerHomogenous + ) { + this.cmpName = cmpName; + this.srcVer = srcVer; + this.targetVer = targetVer; + this.isClusterVerHomogenous = isClusterVerHomogenous; + } + + /** */ + boolean isCompatible(IgniteProductVersion joiningNodeCmpVer) { + if (isClusterVerHomogenous) + return srcVer == null || srcVer.compareTo(joiningNodeCmpVer) <= 0; + + return joiningNodeCmpVer.equals(srcVer) || joiningNodeCmpVer.equals(targetVer); + } + + /** {@inheritDoc} */ + @Override public String toString() { + StringBuilder sb = new StringBuilder("[componentName="); + + sb.append(cmpName); + + sb.append(", compatibleComponentVersions="); + + if (isClusterVerHomogenous) + if (srcVer == null) + sb.append("[any]"); + else + sb.append("[").append(semanticVersion(srcVer)).append(" or greater]"); + else { + sb.append("["); + + if (srcVer != null) + sb.append(srcVer); + + if (targetVer != null) { + if (srcVer != null) + sb.append(", "); + + sb.append(targetVer); + } + + sb.append("]"); + } + + sb.append("]"); + + return sb.toString(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeNodeData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeClusterData.java similarity index 78% rename from modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeNodeData.java rename to modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeClusterData.java index e13b5363cc6f3..f14982edde2b2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeNodeData.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeClusterData.java @@ -17,31 +17,36 @@ package org.apache.ignite.internal.processors.rollingupgrade; +import java.util.Collection; import org.apache.ignite.internal.Order; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteProductFeatures; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteComponentFeatures; import org.apache.ignite.plugin.extensions.communication.Message; /** */ -public class RollingUpgradeNodeData implements Message { +public class RollingUpgradeClusterData implements Message { /** */ @Order(0) boolean isVersionUpgradeEnabled; /** */ @Order(1) - IgniteProductFeatures activeFeatures; + Collection activeFeatures; /** */ @Order(2) boolean isNodeFenceActive; /** */ - public RollingUpgradeNodeData() { + public RollingUpgradeClusterData() { // No-op. } /** */ - public RollingUpgradeNodeData(boolean isVersionUpgradeEnabled, boolean isNodeFenceActive, IgniteProductFeatures activeFeatures) { + public RollingUpgradeClusterData( + boolean isVersionUpgradeEnabled, + boolean isNodeFenceActive, + Collection activeFeatures + ) { this.isVersionUpgradeEnabled = isVersionUpgradeEnabled; this.isNodeFenceActive = isNodeFenceActive; this.activeFeatures = activeFeatures; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeProcessor.java index fe2dd268d9031..9602525255113 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeProcessor.java @@ -17,12 +17,17 @@ package org.apache.ignite.internal.processors.rollingupgrade; +import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.UUID; +import java.util.stream.Collectors; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.cluster.ClusterNode; @@ -32,10 +37,12 @@ import org.apache.ignite.internal.IgniteVersionUtils; import org.apache.ignite.internal.processors.GridProcessorAdapter; import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteComponentFeatures; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeature; import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureManager; import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureSet; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteProductFeatures; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatures; import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeaturesRegistry; import org.apache.ignite.internal.util.distributed.DistributedProcess; import org.apache.ignite.internal.util.distributed.InitMessage; @@ -55,7 +62,6 @@ import static org.apache.ignite.events.EventType.EVT_NODE_VALIDATION_FAILED; import static org.apache.ignite.internal.GridComponent.DiscoveryDataExchangeType.ROLLING_UPGRADE_PROC; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_FEATURES; -import static org.apache.ignite.internal.IgniteVersionUtils.semanticVersion; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RU_ABORT_VERSION_FINALIZATION; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RU_COMPLETE_VERSION_FINALIZATION; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RU_ENABLE; @@ -90,20 +96,21 @@ public class RollingUpgradeProcessor extends GridProcessorAdapter implements Dis /** */ public RollingUpgradeProcessor(GridKernalContext ctx) { - this( - ctx, - new IgniteProductFeatures(IgniteVersionUtils.VER, IgniteFeatureSet.buildFrom(SupportedFeaturesRegistry.class)) + this(ctx, new IgniteComponentFeatures( + IgniteCoreFeature.COMPONENT_NAME, + IgniteVersionUtils.VER, + IgniteFeatureSet.buildFrom(SupportedFeaturesRegistry.class)) ); } /** */ - protected RollingUpgradeProcessor(GridKernalContext ctx, IgniteProductFeatures locVerFeatures) { + protected RollingUpgradeProcessor(GridKernalContext ctx, IgniteComponentFeatures coreFeatures) { super(ctx); enableProc = new ClusterVersionUpgradeEnableProcess(); finalizeProc = new ClusterVersionFinalizationProcess(); finalizeAbortProc = new ClusterVersionFinalizationAbortProcess(); - featureMgr = new IgniteFeatureManager(ctx, locVerFeatures); + featureMgr = new IgniteFeatureManager(ctx, coreFeatures); } /** @return Whether nodes running a higher Ignite version are allowed to join the cluster. */ @@ -148,7 +155,7 @@ public void finalizeClusterVersion() throws IgniteCheckedException { finalizeProc.start().get(); if (log.isInfoEnabled()) - log.info("Cluster version was successfully finalized [activeLogicalVer=" + clusterLogicalVersion() + ']'); + log.info("Cluster version was successfully finalized [componentVersions=" + featureMgr.activeFeatures() + ']'); } /** */ @@ -170,9 +177,9 @@ public IgniteFeatureManager features() { } /** */ - RollingUpgradeState state() { + IgniteComponentUpgradeState state(String cmpName) { synchronized (topGuard) { - return detectRollingUpgradeState(); + return detectComponentUpgradeState(clusterFeatures(), cmpName); } } @@ -180,7 +187,10 @@ RollingUpgradeState state() { @Override public void start() throws IgniteCheckedException { ctx.addNodeAttribute( ATTR_IGNITE_FEATURES, - U.marshal(ctx.marshallerContext().jdkMarshaller(), featureMgr.localVersionFeatures().features())); + U.marshal( + ctx.marshallerContext().jdkMarshaller(), + featureMgr.localVersionFeatures().values().toArray(new IgniteComponentFeatures[0])) + ); ctx.event().addLocalEventListener( evt -> { @@ -208,17 +218,17 @@ RollingUpgradeState state() { int cmpId = discoveryDataType().ordinal(); if (!dataBag.commonDataCollectedFor(cmpId)) - dataBag.addGridCommonData(cmpId, collectRollingUpgradeNodeData()); + dataBag.addGridCommonData(cmpId, collectRollingUpgradeClusterData()); } /** {@inheritDoc} */ @Override public void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) { - RollingUpgradeNodeData gridData = data.commonData(); + RollingUpgradeClusterData gridData = data.commonData(); isVerUpgradeEnabled = gridData.isVersionUpgradeEnabled; isNodeFenceActive = gridData.isNodeFenceActive; - featureMgr.onGridDataReceived(gridData.activeFeatures); + featureMgr.onGridDataReceived(new IgniteFeatures(gridData.activeFeatures)); } /** {@inheritDoc} */ @@ -231,50 +241,64 @@ RollingUpgradeState state() { " cluster version finalization process is complete [joiningNode=" + joiningNode + ']'); } - if (isVerUpgradeEnabled) { - RollingUpgradeState state = detectRollingUpgradeState(); + IgniteFeatures joiningNodeFeatures; - if (!state.isCompatible(joiningNode)) { - return new IgniteNodeValidationResult( - joiningNode.id(), - "The joining node version is incompatible with the current state of the cluster version Rolling" + - " Upgrade being in progress. Upgrade the joining node version and retry the node join procedure" + - " [rollingUpgradeState=" + state + - ", joiningNodeVer=" + joiningNode.version() + - ", joiningNode=" + joiningNode + ']'); - } - - IgniteProductFeatures locActiveFeatures = featureMgr.activeFeatures(); - - IgniteProductFeatures joiningNodeProductFeatures; + try { + joiningNodeFeatures = extractNodeFeatures(joiningNode); + } + catch (IgniteCheckedException e) { + return new IgniteNodeValidationResult( + joiningNode.id(), + "Failed to resolve joining node features [joiningNode=" + joiningNode + ", errMsg=" + e.getMessage() + ']'); + } - try { - joiningNodeProductFeatures = extractProductFeatures(joiningNode); - } - catch (IgniteCheckedException e) { + if (isVerUpgradeEnabled) { + if (!joiningNode.isClient() && !joiningNodeFeatures.components().containsAll(featureMgr.activeFeatures().components())) { return new IgniteNodeValidationResult( joiningNode.id(), - "Failed to resolve joining node product features" + - " [joiningNode=" + joiningNode + ", errMsg=" + e.getMessage() + ']'); + "Some components active in the cluster are not configured on the joining server node" + + " [clusterComponents=" + featureMgr.activeFeatures().components() + + ", joiningNodeComponents=" + joiningNodeFeatures.components() + + ", joiningNode=" + joiningNode + ']' + ); } - if (!locActiveFeatures.isUpgradableTo(joiningNodeProductFeatures)) { - return new IgniteNodeValidationResult( - joiningNode.id(), - "Rolling Upgrade is not available between the current cluster logical version and the joining node" + - " product version. Refer to the documentation to determine between which versions of Ignite" + - " a Rolling Upgrade is possible and retry the node join procedure" + - " [clusterLogicalVer=" + locActiveFeatures.version() + - ", joiningNodeVer=" + joiningNode.version() + - ", joiningNode=" + joiningNode + ']'); + Map clusterFeatures = clusterFeatures(); + + for (IgniteComponentFeatures rmtCmpFeatures : joiningNodeFeatures.values()) { + IgniteComponentUpgradeState state = detectComponentUpgradeState(clusterFeatures, rmtCmpFeatures.componentName()); + + if (!state.isCompatible(rmtCmpFeatures.version())) { + return new IgniteNodeValidationResult( + joiningNode.id(), + "The joining node is incompatible with the current state of the version Rolling Upgrade being " + + "in progress. Upgrade the joining node component versions and retry the node join procedure" + + " [rollingUpgradeState=" + state + + ", joiningNodeComponentVer=" + rmtCmpFeatures.version() + + ", joiningNode=" + joiningNode + ']'); + } + + IgniteComponentFeatures locCmpFeatures = featureMgr.activeComponentFeatures(rmtCmpFeatures.componentName()); + + if (locCmpFeatures != null && !locCmpFeatures.isUpgradableTo(rmtCmpFeatures)) { + return new IgniteNodeValidationResult( + joiningNode.id(), + "Ignite component Rolling Upgrade is not supported between the component version active in the cluster " + + "and the version running on the joining node. Refer to the documentation for supported Rolling Upgrade " + + "version combinations, and then retry the node join operation" + + " [componentName=" + locCmpFeatures.componentName() + + ", clusterComponentVer=" + locCmpFeatures.version() + + ", joiningNodeComponentVer=" + rmtCmpFeatures.version() + + ", joiningNode=" + joiningNode + ']'); + } } } - else if (!joiningNode.version().equals(localProductVersion())) { + else if (!joiningNodeComponentVersionsMatchCluster(joiningNode, joiningNodeFeatures)) { return new IgniteNodeValidationResult( joiningNode.id(), - "The joining node version differs from the version of the cluster" + - " [clusterVer=" + localProductVersion() + - ", joiningNodeVer=" + joiningNode.version() + + "One or more component versions on the joining node differ from the corresponding versions active in the cluster" + + " [clusterComponentVers=" + featureMgr.activeFeatures() + + ", joiningNodeComponentVers=" + joiningNodeFeatures + ", joiningNode=" + joiningNode + ']'); } @@ -294,64 +318,97 @@ else if (!joiningNode.version().equals(localProductVersion())) { } /** */ - private IgniteProductVersion localProductVersion() { - return featureMgr.localVersionFeatures().version(); - } + private IgniteComponentUpgradeState detectComponentUpgradeState(Map clusterFeatures, String cmpName) { + assert Thread.holdsLock(topGuard); - /** */ - private IgniteProductVersion clusterLogicalVersion() { - return featureMgr.activeFeatures().version(); - } + SortedSet clusterCmpVersions = distinctClusterComponentVersions(clusterFeatures, cmpName); - /** */ - private RollingUpgradeState detectRollingUpgradeState() { - SortedSet clusterVers = distinctClusterProductVersions(); + assert !clusterCmpVersions.isEmpty() && clusterCmpVersions.size() <= 2; - assert !clusterVers.isEmpty() && clusterVers.size() <= 2; + IgniteProductVersion minCmpVer = clusterCmpVersions.first(); + IgniteProductVersion maxCmpVer = clusterCmpVersions.last(); - IgniteProductVersion minClusterVer = clusterVers.first(); - IgniteProductVersion maxClusterVer = clusterVers.last(); + if (!Objects.equals(minCmpVer, maxCmpVer)) + return new IgniteComponentUpgradeState(cmpName, minCmpVer, maxCmpVer, false); - if (!minClusterVer.equals(maxClusterVer)) - return new RollingUpgradeState(minClusterVer, maxClusterVer, false); + IgniteComponentFeatures activeCompFeatures = featureMgr.activeComponentFeatures(cmpName); - IgniteProductVersion logicalVer = clusterLogicalVersion(); + IgniteProductVersion logicalCmpVer = activeCompFeatures == null ? null : activeCompFeatures.version(); - return new RollingUpgradeState( - logicalVer, - logicalVer.equals(maxClusterVer) ? null : maxClusterVer, + return new IgniteComponentUpgradeState( + cmpName, + logicalCmpVer, + Objects.equals(logicalCmpVer, maxCmpVer) ? null : maxCmpVer, true); } /** */ - private SortedSet distinctClusterProductVersions() { + private SortedSet distinctClusterComponentVersions( + Map + clusterFeatures, String cmpName + ) { assert Thread.holdsLock(topGuard); - TreeSet res = new TreeSet<>(); + SortedSet distinctCmpVersions = new TreeSet<>(Comparator.nullsFirst(Comparator.naturalOrder())); - for (ClusterNode node : ctx.discovery().discoverySpiRemoteNodes()) - res.add(node.version()); + clusterFeatures.forEach((node, nodeFeatures) -> { + IgniteComponentFeatures cmpFeatures = nodeFeatures.componentFeatures(cmpName); - res.add(ctx.discovery().localNode().version()); + if (node.isClient() && cmpFeatures == null) + return; // Components are optional on client nodes, even when they are configured on servers. - for (ClusterNode node : joiningNodes) - res.add(node.version()); + distinctCmpVersions.add(cmpFeatures == null ? null : cmpFeatures.version()); + }); - return res; + return distinctCmpVersions; } /** */ - private RollingUpgradeNodeData collectRollingUpgradeNodeData() { - return new RollingUpgradeNodeData(isVerUpgradeEnabled, isNodeFenceActive, featureMgr.activeFeatures()); + private boolean joiningNodeComponentVersionsMatchCluster(ClusterNode joiningNode, IgniteFeatures joiningNodeFeatures) { + IgniteFeatures locFeatures = featureMgr.localVersionFeatures(); + + return joiningNode.isClient() + ? locFeatures.containsAll(joiningNodeFeatures) + : locFeatures.equals(joiningNodeFeatures); + } + + /** */ + private boolean isReadyForVersionFinalization() { + Map clusterFeatures = clusterFeatures(); + + Set distinctFeaturesOnServers = new HashSet<>(); + Set distinctFeaturesOnClients = new HashSet<>(); + + clusterFeatures.forEach((node, features) -> { + if (!node.isClient()) + distinctFeaturesOnServers.add(features); + else + distinctFeaturesOnClients.add(features); + }); + + if (distinctFeaturesOnServers.size() != 1) + return false; + + IgniteFeatures srvFeatures = F.first(distinctFeaturesOnServers); + + return distinctFeaturesOnClients.stream().allMatch(srvFeatures::containsAll); } /** */ - private IgniteProductFeatures extractProductFeatures(ClusterNode node) throws IgniteCheckedException { + private RollingUpgradeClusterData collectRollingUpgradeClusterData() { + return new RollingUpgradeClusterData(isVerUpgradeEnabled, isNodeFenceActive, featureMgr.activeFeatures().values()); + } + + /** */ + private IgniteFeatures extractNodeFeatures(ClusterNode node) throws IgniteCheckedException { byte[] attrVal = node.attribute(ATTR_IGNITE_FEATURES); - IgniteFeatureSet features = U.unmarshal(ctx.marshallerContext().jdkMarshaller(), attrVal, U.resolveClassLoader(ctx.config())); + IgniteComponentFeatures[] nodeFeatures = U.unmarshal( + ctx.marshallerContext().jdkMarshaller(), + attrVal, + U.resolveClassLoader(ctx.config())); - return new IgniteProductFeatures(node.version(), features); + return new IgniteFeatures(List.of(nodeFeatures)); } /** */ @@ -359,6 +416,35 @@ private static Throwable firstError(Map errors) { return F.isEmpty(errors) ? null : F.firstValue(errors); } + /** */ + private Map clusterFeatures() { + try { + Map res = new HashMap<>(); + + for (ClusterNode node : clusterNodes()) + res.put(node, extractNodeFeatures(node)); + + return res; + } + catch (IgniteCheckedException e) { + U.error(log, "Failed to resolve cluster features", e); + + throw new IgniteException("Failed to resolve cluster features", e); + } + } + + /** */ + private Set clusterNodes() { + assert Thread.holdsLock(topGuard); + + Set clusterNodes = new HashSet<>(ctx.discovery().discoverySpiRemoteNodes()); + + clusterNodes.add(ctx.discovery().localNode()); + clusterNodes.addAll(joiningNodes); + + return clusterNodes; + } + /** */ private class ClusterVersionUpgradeEnableProcess extends AbstractProcess { /** */ @@ -480,15 +566,13 @@ private IgniteInternalFuture executePreparePhase(UUID reqId, Message re activeProcId = reqId; synchronized (topGuard) { - Set distinctNodeVersions = distinctClusterProductVersions(); - - if (distinctNodeVersions.size() > 1) { + if (!isReadyForVersionFinalization()) return new GridFinishedFuture<>(new IgniteException( - "Cluster version finalization failed. The topology contains nodes running multiple different" + - " versions. Retry the operation after all cluster nodes are upgraded to the same version " + - "[distinctNodeVersions=" + distinctNodeVersions + "]" - )); - } + "Cluster version finalization failed. The cluster contains nodes running" + + " different versions of one or more components. Retry the operation after upgrading" + + " all cluster node components to the same version" + + " [clusterFeatures=" + clusterFeatures().entrySet().stream().collect( + Collectors.toMap(e -> e.getKey().id(), Map.Entry::getValue)) + ']')); isNodeFenceActive = true; @@ -515,10 +599,8 @@ private IgniteInternalFuture executeCompletePhase(UUID reqId, Message r // after the prepare phase has completed successfully but before the completion phase begins. // Aborting cluster version finalization is mutually exclusive with the finalization completion // phase and is applied consistently across all cluster nodes. - return new GridFinishedFuture<>(new IgniteException( - "Failed to complete the cluster version finalization operation." + - " The operation may have been aborted by an administrator. Retry the operation if possible" - )); + return new GridFinishedFuture<>(new IgniteException("Failed to complete the cluster version finalization" + + " operation. The operation may have been aborted by an administrator. Retry the operation if possible")); } featureMgr.activateLocalVersionFeatures(); @@ -573,54 +655,4 @@ private void finish(UUID reqId, Map responses, Map features(); +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteCoreFeature.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteCoreFeature.java index 07476bf5501e8..a0250ef8a6237 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteCoreFeature.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteCoreFeature.java @@ -24,6 +24,9 @@ /** Represents an implementation of {@link IgniteFeature} used to define incompatible changes in Ignite core functionality. */ public class IgniteCoreFeature implements IgniteFeature { + /** */ + public static final String COMPONENT_NAME = "core"; + /** */ @GridToStringInclude private final int id; @@ -40,6 +43,11 @@ public IgniteCoreFeature(int id) { return id; } + /** {@inheritDoc} */ + @Override public String componentName() { + return COMPONENT_NAME; + } + /** {@inheritDoc} */ @Override public boolean equals(Object o) { if (this == o) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeature.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeature.java index a704e967e7384..e1882675ecc84 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeature.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeature.java @@ -30,6 +30,7 @@ * they are handled consistently throughout the Rolling Upgrade process. * * @see IgniteFeatureSet + * @see IgniteComponentFeaturesProvider */ public interface IgniteFeature { /** @@ -42,4 +43,7 @@ public interface IgniteFeature { * @return The unique identifier of this feature. */ int id(); + + /** The name of the Ignite component to which this {@link IgniteFeature} belongs. */ + String componentName(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatureManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatureManager.java index 744fc102be3d7..9242613c9f204 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatureManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatureManager.java @@ -17,39 +17,51 @@ package org.apache.ignite.internal.processors.rollingupgrade.feature; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import org.apache.ignite.IgniteException; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.lang.IgniteRunnable; +import org.jetbrains.annotations.Nullable; -/** Maintains the set of active cluster {@link IgniteProductFeatures} used by Rolling Upgrade logic. */ +/** Maintains the set of active cluster {@link IgniteComponentFeatures} used by Rolling Upgrade logic. */ public class IgniteFeatureManager { /** */ private final GridKernalContext ctx; /** */ - private final IgniteProductFeatures locVerFeatures; + private final IgniteFeatures locVerFeatures; /** */ private final GridFutureAdapter locVerFeaturesActivationFut; /** */ - private volatile IgniteProductFeatures activeFeatures; + private volatile IgniteFeatures activeFeatures; /** */ - public IgniteFeatureManager(GridKernalContext ctx, IgniteProductFeatures locVerFeatures) { + public IgniteFeatureManager(GridKernalContext ctx, IgniteComponentFeatures locCoreFeatures) { this.ctx = ctx; - this.locVerFeatures = locVerFeatures; + this.locVerFeatures = collectLocalVersionFeatures(ctx, locCoreFeatures); locVerFeaturesActivationFut = new GridFutureAdapter<>(); } /** @return The set of features declared by the local node's product version. */ - public IgniteProductFeatures localVersionFeatures() { + public IgniteFeatures localVersionFeatures() { return locVerFeatures; } + /** @return Active functions of the specified component. */ + @Nullable public IgniteComponentFeatures activeComponentFeatures(String cmpName) { + return activeFeatures().componentFeatures(cmpName); + } + /** @return The set of features currently active in the cluster. */ - public IgniteProductFeatures activeFeatures() { - final IgniteProductFeatures finalActiveFeatures = activeFeatures; + public IgniteFeatures activeFeatures() { + final IgniteFeatures finalActiveFeatures = activeFeatures; checkActiveFeaturesInitialized(finalActiveFeatures); @@ -58,7 +70,7 @@ public IgniteProductFeatures activeFeatures() { /** @return {@code true} if the specified {@link IgniteFeature} is active in the cluster; {@code false} otherwise. */ public boolean isActive(IgniteFeature feature) { - final IgniteProductFeatures finalActiveFeatures = activeFeatures; + final IgniteFeatures finalActiveFeatures = activeFeatures; checkActiveFeaturesInitialized(finalActiveFeatures); @@ -69,7 +81,7 @@ public boolean isActive(IgniteFeature feature) { public void listenActivation(IgniteFeature feature, IgniteRunnable lsnr) { assert locVerFeatures.contains(feature); - final IgniteProductFeatures finalActiveFeatures = activeFeatures; + final IgniteFeatures finalActiveFeatures = activeFeatures; checkActiveFeaturesInitialized(finalActiveFeatures); @@ -80,7 +92,7 @@ public void listenActivation(IgniteFeature feature, IgniteRunnable lsnr) { } /** */ - public void onGridDataReceived(IgniteProductFeatures activeClusterFeatures) { + public void onGridDataReceived(IgniteFeatures activeClusterFeatures) { if (locVerFeatures.equals(activeClusterFeatures)) activateLocalVersionFeatures(); else @@ -104,10 +116,48 @@ public synchronized void activateLocalVersionFeatures() { } /** */ - private void checkActiveFeaturesInitialized(IgniteProductFeatures activeFeatures) { + private void checkActiveFeaturesInitialized(IgniteFeatures activeFeatures) { if (activeFeatures == null) { throw new IllegalStateException("Local node features are not yet initialized [locNodeId=" + ctx.discovery().localNode().id() + ']'); } } + + /** */ + private IgniteFeatures collectLocalVersionFeatures(GridKernalContext ctx, IgniteComponentFeatures locCoreFeatures) { + Set features = new HashSet<>(); + + features.add(locCoreFeatures); + + IgniteComponentFeaturesProvider[] components = ctx.plugins().extensions(IgniteComponentFeaturesProvider.class); + + if (!F.isEmpty(components)) { + for (IgniteComponentFeaturesProvider component : components) + features.add(buildComponentFeatures(component)); + } + + return new IgniteFeatures(features); + } + + /** */ + private IgniteComponentFeatures buildComponentFeatures(IgniteComponentFeaturesProvider cmpFeaturesProvider) { + Collection cmpFeatures = cmpFeaturesProvider.features(); + + A.notEmpty(cmpFeatures, "component features"); + + boolean allFeaturesBelongToComponent = cmpFeatures.stream() + .map(IgniteFeature::componentName) + .allMatch(featureCmp -> featureCmp.equals(cmpFeaturesProvider.componentName())); + + if (!allFeaturesBelongToComponent) { + throw new IgniteException("All specified Ignite Features must belong to the current component" + + " [componentName=" + cmpFeaturesProvider.componentName() + ']'); + } + + return new IgniteComponentFeatures( + cmpFeaturesProvider.componentName(), + cmpFeaturesProvider.componentVersion(), + IgniteFeatureSet.buildFrom(cmpFeatures) + ); + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatureSet.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatureSet.java index e6d5af32b832c..aa994965bc221 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatureSet.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatureSet.java @@ -252,10 +252,10 @@ public static IgniteFeatureSet buildFrom(Class cls) { *

{@link IgniteFeature} instances that are not present in the specified collection and whose IDs are lower than * the minimum ID among the specified {@link IgniteFeature}s are considered non-deactivatable.

*/ - public static IgniteFeatureSet buildFrom(Collection nodeFeatures) { - A.notEmpty(nodeFeatures, "node features"); + public static IgniteFeatureSet buildFrom(Collection features) { + A.notEmpty(features, "features"); - GridIntList featureIds = new GridIntList(nodeFeatures.stream().mapToInt(IgniteFeature::id).toArray()); + GridIntList featureIds = new GridIntList(features.stream().mapToInt(IgniteFeature::id).toArray()); featureIds.sort(); @@ -281,7 +281,7 @@ public static IgniteFeatureSet buildFrom(Collection nod } /** */ - static Collection readDeclaredFeatures(Class cls) { + public static Collection readDeclaredFeatures(Class cls) { A.notNull(cls, "cls"); List features = new ArrayList<>(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatures.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatures.java new file mode 100644 index 0000000000000..f6c5bf7b98966 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatures.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rollingupgrade.feature; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.util.tostring.GridToStringExclude; +import org.jetbrains.annotations.Nullable; + +/** + * Represents a set of {@link IgniteFeature}s supported by an Ignite instance. Ignite is divided into independent components. + * Each component is associated with its version and a set of {@link IgniteFeature}s. + */ +public class IgniteFeatures { + /** */ + @GridToStringExclude + private final Map features; + + /** */ + public IgniteFeatures(Collection features) { + this.features = indexByComponentName(features); + } + + /** */ + public Set components() { + return Collections.unmodifiableSet(features.keySet()); + } + + /** */ + public Collection values() { + return Collections.unmodifiableCollection(features.values()); + } + + /** */ + @Nullable public IgniteComponentFeatures componentFeatures(String cmpName) { + return features.get(cmpName); + } + + /** */ + public boolean containsAll(IgniteFeatures other) { + if (!components().containsAll(other.components())) + return false; + + for (IgniteComponentFeatures otherCmpFeatures : other.features.values()) { + if (!otherCmpFeatures.equals(features.get(otherCmpFeatures.componentName()))) + return false; + } + + return true; + } + + /** */ + public boolean contains(IgniteFeature feature) { + IgniteComponentFeatures cmpFeatures = features.get(feature.componentName()); + + return cmpFeatures != null && cmpFeatures.contains(feature.id()); + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) + return false; + + IgniteFeatures other = (IgniteFeatures)o; + + return Objects.equals(features, other.features); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return Objects.hashCode(features); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return features.values().stream().map(IgniteComponentFeatures::toString).collect(Collectors.joining(", ", "[", "]")); + } + + /** */ + private static Map indexByComponentName(Collection features) { + Map res = new HashMap<>(); + + for (IgniteComponentFeatures compFeatures : features) { + if (res.put(compFeatures.componentName(), compFeatures) != null) + throw new IgniteException("Duplicated component name [cmpName=" + compFeatures.componentName() + ']'); + } + + return res; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteProductFeatures.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteProductFeatures.java deleted file mode 100644 index 8031306ef091b..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteProductFeatures.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.rollingupgrade.feature; - -import java.util.Objects; -import org.apache.ignite.internal.Order; -import org.apache.ignite.lang.IgniteProductVersion; -import org.apache.ignite.plugin.extensions.communication.Message; - -/** Represents a set of {@link IgniteFeature}s available for the specific Ignite product version. */ -public class IgniteProductFeatures implements Message { - /** */ - @Order(0) - IgniteProductVersion ver; - - /** */ - @Order(1) - IgniteFeatureSet features; - - /** */ - public IgniteProductFeatures() { - // No-op. - } - - /** */ - public IgniteProductFeatures(IgniteProductVersion ver, IgniteFeatureSet features) { - this.ver = ver; - this.features = features; - } - - /** */ - public IgniteProductVersion version() { - return ver; - } - - /** */ - public IgniteFeatureSet features() { - return features; - } - - /** */ - public boolean contains(IgniteFeature feature) { - return features.contains(feature.id()); - } - - /** */ - public boolean isUpgradableTo(IgniteProductFeatures target) { - return features.isUpgradableTo(target.features); - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) - return false; - - IgniteProductFeatures other = (IgniteProductFeatures)o; - - return Objects.equals(features, other.features) && Objects.equals(ver, other.ver); - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return Objects.hash(features, ver); - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java new file mode 100644 index 0000000000000..0837b775796bf --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java @@ -0,0 +1,648 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rollingupgrade; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.Ignition; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.events.Event; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgnitionEx; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener; +import org.apache.ignite.internal.managers.eventstorage.HighPriorityListener; +import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteComponentFeatures; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteComponentFeaturesProvider; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeature; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureSet; +import org.apache.ignite.internal.processors.rollingupgrade.feature.TestIgniteReleaseFeatures_2_18_0; +import org.apache.ignite.internal.processors.rollingupgrade.feature.TestPluginComponentFeaturesProvider; +import org.apache.ignite.internal.processors.rollingupgrade.feature.TestPluginFeature; +import org.apache.ignite.internal.processors.rollingupgrade.feature.TestPluginReleaseFeatures_1_0_0; +import org.apache.ignite.internal.util.lang.ConsumerX; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteProductVersion; +import org.apache.ignite.lang.IgniteRunnable; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.ExtensionRegistry; +import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.spi.IgniteNodeValidationResult; +import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jspecify.annotations.Nullable; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.ignite.events.EventType.EVT_NODE_VALIDATION_FAILED; +import static org.apache.ignite.internal.IgniteVersionUtils.semanticVersion; +import static org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature.COMPONENT_NAME; + +/** + * Provides the ability to override a node's version and supported {@link IgniteFeature}s in order to + * simulate a Rolling Upgrade procedure. + * + *

For testing purposes, the following "fake" Ignite versions and their corresponding + * {@link IgniteFeature}s have been introduced. These versions are used solely for testing and do not + * correspond to any actual Ignite releases. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
VersionFeatures
2.18.0not supported
2.19.0{@code IgniteFeatureSet [0]}
2.19.1{@code IgniteFeatureSet [0, 1]}
2.19.2{@code IgniteFeatureSet [0 -> 2]}
2.19.3{@code IgniteFeatureSet [0 -> 2, 6]}
2.20.0{@code IgniteFeatureSet [0 -> 4]}
2.20.1{@code IgniteFeatureSet [0 -> 4, 6]}
2.21.0{@code IgniteFeatureSet [3 -> 6]}
2.21.1{@code IgniteFeatureSet [3 -> 7]}
+ */ +public abstract class AbstractRollingUpgradeTest extends GridCommonAbstractTest { + /** */ + protected static final String TEST_DEFAULT_VER = "2.19.0"; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return getConfiguration(igniteInstanceName, TEST_DEFAULT_VER); + } + + /** */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + TestRollingUpgradeProcessor.nodeJoinValidationCompletedLatch = null; + TestRollingUpgradeProcessor.nodeJoinUnblockedLatch = null; + TestNodeValidationFailedEventListener.nodeValidationFailedEventUnblockedLatch = null; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** */ + protected IgniteConfiguration getConfiguration(int idx, String ver) throws Exception { + return getConfiguration(getTestIgniteInstanceName(idx), ver); + } + + /** */ + protected IgniteConfiguration getConfiguration(String igniteInstanceName, String ver) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setCommunicationSpi(new TestRecordingCommunicationSpi()); + + TestVersions versions = TestVersions.parse(ver); + + IgniteComponentFeatures testCoreFeatures = new IgniteComponentFeatures( + COMPONENT_NAME, + IgniteProductVersion.fromString(versions.coreVersion()), + IgniteFeatureSet.buildFrom(readDeclaredCoreFeatures(versions.coreVersion())) + ); + + cfg.setPluginProviders(new AbstractTestPluginProvider() { + @Override public String name() { + return "test-rolling-upgrade-processor-provider"; + } + + /** {@inheritDoc} */ + @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { + if (versions.containsPlugin()) + registry.registerExtension( + IgniteComponentFeaturesProvider.class, + new TestPluginComponentFeaturesProvider(versions.pluginVersion())); + } + + /** {@inheritDoc} */ + @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { + if (cls.isAssignableFrom(DiscoveryNodeValidationProcessor.class)) + return (T)new TestRollingUpgradeProcessor(((IgniteEx)ctx.grid()).context(), testCoreFeatures); + + return null; + } + }); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi() { + @Override public void setNodeAttributes(Map attrs, IgniteProductVersion ignored) { + super.setNodeAttributes(attrs, IgniteProductVersion.fromString(versions.coreVersion())); + } + }; + + discoSpi.setIpFinder(((TcpDiscoverySpi)cfg.getDiscoverySpi()).getIpFinder()); + + cfg.setDiscoverySpi(discoSpi); + + return cfg; + } + + /** */ + protected IgniteEx startGrid(int idx, String ver) throws Exception { + return startGrid(getConfiguration(idx, ver)); + } + + /** */ + protected IgniteEx startClientGrid(int idx, String ver) throws Exception { + return startClientGrid(getConfiguration(idx, ver)); + } + + /** */ + protected IgniteEx startGrid(int idx, String ver, boolean isClient) throws Exception { + return isClient ? startClientGrid(idx, ver) : startGrid(idx, ver); + } + + /** */ + public static Collection readDeclaredCoreFeatures(String ver) throws Exception { + Class cls = Class.forName( + TestIgniteReleaseFeatures_2_18_0.class.getPackageName() + ".TestIgniteReleaseFeatures_" + ver.replace(".", "_")); + + return IgniteFeatureSet.readDeclaredFeatures(cls); + } + + /** */ + public static Collection readDeclaredPluginFeatures(String ver) throws Exception { + Class cls = Class.forName( + TestPluginReleaseFeatures_1_0_0.class.getPackageName() + ".TestPluginReleaseFeatures_" + ver.replace(".", "_")); + + return IgniteFeatureSet.readDeclaredFeatures(cls); + } + + /** */ + protected boolean isFeatureActive(Ignite ignite, IgniteFeature feature) { + return ((IgniteEx)ignite).context().rollingUpgrade().features().isActive(feature); + } + + /** */ + protected void listenFeatureActivation(Ignite ignite, IgniteFeature feature, IgniteRunnable lsnr) { + ((IgniteEx)ignite).context().rollingUpgrade().features().listenActivation(feature, lsnr); + } + + /** */ + protected void startCluster() throws Exception { + startCluster(TEST_DEFAULT_VER); + } + + /** */ + protected void startCluster(String ver) throws Exception { + startGrid(0, ver); + startGrid(1, ver); + startClientGrid(2, ver); + } + + /** */ + protected UUID extractActiveFinalizeProcessId(int nodeIdx) { + Object enable = U.field(grid(nodeIdx).context().rollingUpgrade(), "finalizeProc"); + + return U.field(enable, "locInitOpId"); + } + + /** */ + protected void checkVersionFinalizationFailed(int initiatorNodeIdx, String msg) { + GridTestUtils.assertThrowsAnyCause( + log, + () -> { + ru(initiatorNodeIdx).finalizeClusterVersion(); + + return null; + }, + IgniteCheckedException.class, + msg + ); + } + + /** */ + protected void checkJoinSuccess(int nodeIdx, boolean isClient) throws Exception { + checkJoinSuccess(nodeIdx, TEST_DEFAULT_VER, isClient); + } + + /** */ + protected void checkJoinSuccess(int nodeIdx, String ver, boolean isClient) throws Exception { + int clusterSize = clusterNode().cluster().nodes().size(); + + startGrid(nodeIdx, ver, isClient); + + assertEquals(clusterSize + 1, clusterNode().cluster().nodes().size()); + } + + /** */ + protected void checkJoinFailed(int nodeIdx, String ver, String msg) { + checkJoinFailed(nodeIdx, ver, true, msg); + } + + /** */ + protected void checkJoinFailed(int nodeIdx, String ver, boolean checkClientNode, String msg) { + int expClusterSize = clusterNode().cluster().nodes().size(); + + GridTestUtils.assertThrowsAnyCause(log, () -> startGrid(nodeIdx, ver), IgniteSpiException.class, msg); + + if (checkClientNode) + GridTestUtils.assertThrowsAnyCause(log, () -> startClientGrid(nodeIdx, ver), IgniteSpiException.class, msg); + + assertEquals(expClusterSize, clusterNode().cluster().nodes().size()); + } + + /** */ + protected IgniteEx clusterNode() { + return (IgniteEx)Ignition.allGrids().stream().filter(i -> !i.cluster().localNode().isClient()).findFirst().orElseThrow(); + } + + /** */ + protected void checkVersionUpgradeInProgress(String srcVer, String targetVer) throws Exception { + checkVersionUpgradeInProgress(srcVer, srcVer, targetVer); + } + + /** */ + protected void checkVersionUpgradeInProgress(String logicalVer, String srcVer, String targetVer) throws Exception { + checkVersionUpgradeEnabledStatus(true); + checkRollingUpgradeState(logicalVer, srcVer, targetVer); + + checkFeaturesActive(logicalVer); + + TestVersions logicalVersions = TestVersions.parse(logicalVer); + TestVersions targetVersions = TestVersions.parse(targetVer); + + checkFeaturesNotActive(newFeatures( + logicalVersions.coreVersion() == null ? Collections.emptyList() : readDeclaredCoreFeatures(logicalVersions.coreVersion()), + targetVersions.coreVersion() == null ? Collections.emptyList() : readDeclaredCoreFeatures(targetVersions.coreVersion())) + ); + + checkFeaturesNotActive(newFeatures( + logicalVersions.containsPlugin() ? readDeclaredPluginFeatures(logicalVersions.pluginVersion()) : Collections.emptyList(), + targetVersions.containsPlugin() ? readDeclaredPluginFeatures(targetVersions.pluginVersion()) : Collections.emptyList()) + ); + } + + /** */ + protected void checkVersionUpgradeInactive(String expVer) throws Exception { + checkVersionUpgradeEnabledStatus(false); + checkFeaturesActive(expVer); + } + + /** */ + protected void checkVersionUpgradeEnabledStatus(boolean enabled) { + List cluster = Ignition.allGrids(); + + for (Ignite ignite : cluster) + assertEquals(enabled, ru(ignite).isVersionUpgradeEnabled()); + } + + /** */ + protected void checkFeaturesActive(String ver) throws Exception { + TestVersions versions = TestVersions.parse(ver); + + if (versions.coreVersion() != null) + checkFeaturesActive(readDeclaredCoreFeatures(versions.coreVersion()), false); + + if (versions.containsPlugin()) + checkFeaturesActive(readDeclaredPluginFeatures(versions.pluginVersion()), true); + } + + /** */ + protected void checkFeaturesActive(Collection features, boolean ignoreClients) { + List cluster = Ignition.allGrids(); + + AtomicInteger activationNotifiedCnt = new AtomicInteger(); + int expFeaturesCnt = 0; + + for (Ignite ignite : cluster) { + for (IgniteFeature feature : features) { + if (ignoreClients + && ignite.configuration().isClientMode() + && !ru(ignite).features().localVersionFeatures().components().contains(feature.componentName()) + ) + continue; + + assertTrue(isFeatureActive(ignite, feature)); + listenFeatureActivation(ignite, feature, activationNotifiedCnt::incrementAndGet); + + ++expFeaturesCnt; + } + } + + assertEquals(expFeaturesCnt, activationNotifiedCnt.get()); + } + + /** */ + protected void checkFeaturesNotActive(Collection features) { + if (F.isEmpty(features)) + return; + + for (Ignite ignite : Ignition.allGrids()) { + for (IgniteFeature feature : features) + assertFalse(isFeatureActive(ignite, feature)); + } + } + + /** */ + protected Collection newFeatures(Collection srcFeatures, Collection targetFeatures) { + List res = new ArrayList<>(); + + for (IgniteFeature targetFeature : targetFeatures) { + if (!srcFeatures.contains(targetFeature)) + res.add(targetFeature); + } + + return res; + } + + /** */ + protected void checkFeatureActivationSubscription(int nodeIdx, IgniteFeature feature, CountDownLatch featureActivationLatch) { + long expCnt = featureActivationLatch.getCount(); + + assertFalse(isFeatureActive(grid(nodeIdx), feature)); + listenFeatureActivation(grid(nodeIdx), feature, featureActivationLatch::countDown); + assertEquals(expCnt, featureActivationLatch.getCount()); + } + + /** */ + protected void upgradeNodeVersion(int nodeIdx, String targetVer) throws Exception { + upgradeNodeVersion(nodeIdx, TEST_DEFAULT_VER, targetVer); + } + + /** */ + protected void upgradeNodeVersion(int nodeIdx, String srcVer, String targetVer) throws Exception { + upgradeNodeVersion(nodeIdx, srcVer, srcVer, targetVer); + } + + /** */ + protected void upgradeNodeVersion(int nodeIdx, String logicalVer, String srcVer, String targetVer) throws Exception { + boolean isClient = grid(nodeIdx).context().clientNode(); + + stopGrid(nodeIdx); + checkJoinSuccess(nodeIdx, targetVer, isClient); + checkVersionUpgradeInProgress(logicalVer, srcVer, targetVer); + } + + /** */ + protected void finalizeClusterVersion(int nodeIdx, String expVer) throws Exception { + ru(nodeIdx).finalizeClusterVersion(); + + checkVersionUpgradeInactive(expVer); + } + + /** */ + protected void restartNode(int nodeIdx) throws Exception { + String ver = resolveNodeCompoundVersion(nodeIdx); + boolean isClient = grid(nodeIdx).context().clientNode(); + + stopGrid(nodeIdx); + checkJoinSuccess(nodeIdx, ver, isClient); + } + + /** */ + protected void forAllNodes(ConsumerX nodeProcessor) throws Exception { + for (Ignite ignite : Ignition.allGrids()) + nodeProcessor.accept(getTestIgniteInstanceIndex(ignite.name())); + } + + /** */ + protected void checkUpgradeFailed(int nodeIdx, String targetVer, String errMsg) throws Exception { + String srcVer = resolveNodeCompoundVersion(nodeIdx); + boolean isClient = grid(nodeIdx).context().clientNode(); + + stopGrid(nodeIdx); + + GridTestUtils.assertThrowsAnyCause(log, () -> startGrid(nodeIdx, targetVer, isClient), IgniteSpiException.class, errMsg); + + startGrid(nodeIdx, srcVer, isClient); + } + + /** */ + String resolveNodeCompoundVersion(int nodeIdx) { + return ru(nodeIdx).features().localVersionFeatures().values().stream() + .sorted(Comparator.comparing(IgniteComponentFeatures::componentName)) + .map(f -> semanticVersion(f.version())) + .collect(Collectors.joining("|")); + } + + /** */ + protected void checkRollingUpgradeState(String expLogicalVer, String expSrcVer, String expTargetVer) { + TestVersions expLogicalVersions = TestVersions.parse(expLogicalVer); + TestVersions expSrcVersions = TestVersions.parse(expSrcVer); + TestVersions expTargetVersions = TestVersions.parse(expTargetVer); + + for (Ignite ignite : Ignition.allGrids()) { + assertTrue(ru(ignite).isVersionUpgradeEnabled()); + + expLogicalVersions.cmpVersions.forEach((cmp, ver) -> { + IgniteComponentFeatures cmpFeatures = ru(ignite).features().activeComponentFeatures(cmp); + + assertEquals(ver, cmpFeatures == null ? null : semanticVersion(cmpFeatures.version())); + }); + + expSrcVersions.cmpVersions.forEach((cmp, ver) -> { + IgniteComponentUpgradeState state = ru(ignite).state(cmp); + + assertEquals(ver, state.srcVer == null ? null : semanticVersion(state.srcVer)); + }); + + expTargetVersions.cmpVersions.forEach((cmp, ver) -> { + IgniteComponentUpgradeState state = ru(ignite).state(cmp); + + String expVer = Objects.equals(expSrcVersions.cmpVersions.get(cmp), ver) ? null : ver; + + assertEquals(expVer, state.targetVer == null ? null : semanticVersion(state.targetVer)); + }); + } + } + + /** */ + protected int coordinatorIndex() { + for (Ignite grid : IgnitionEx.allGridsx()) { + if (U.isLocalNodeCoordinator(((IgniteEx)grid).context().discovery())) + return getTestIgniteInstanceIndex(grid.name()); + } + + fail("Failed to resolve coordinator node"); + + return -1; + } + + /** */ + protected RollingUpgradeProcessor ru(int nodeIdx) { + return ru(grid(nodeIdx)); + } + + /** */ + protected static RollingUpgradeProcessor ru(Ignite ignite) { + return ((IgniteEx)ignite).context().rollingUpgrade(); + } + + /** */ + protected static class TestVersions { + /** */ + private final Map cmpVersions = new HashMap<>(); + + /** */ + public TestVersions(List cmpVersions) { + assertTrue(!cmpVersions.isEmpty()); + assertTrue(cmpVersions.size() <= 2); + + this.cmpVersions.put(IgniteCoreFeature.COMPONENT_NAME, cmpVersions.get(0)); + + if (cmpVersions.size() > 1) + this.cmpVersions.put(TestPluginFeature.COMPONENT_NAME, cmpVersions.get(1)); + } + + /** */ + public static TestVersions parse(String ver) { + if (ver == null) + return new TestVersions(Collections.singletonList(null)); + + return new TestVersions(Arrays.stream(ver.split("\\|")) + .map(String::trim) + .map(v -> "null".equals(v) ? null : v) + .collect(Collectors.toList())); + } + + /** */ + public String pluginVersion() { + return cmpVersions.get(TestPluginFeature.COMPONENT_NAME); + } + + /** */ + public String coreVersion() { + return cmpVersions.get(IgniteCoreFeature.COMPONENT_NAME); + } + + /** */ + public boolean containsPlugin() { + return cmpVersions.get(TestPluginFeature.COMPONENT_NAME) != null; + } + } + + /** */ + protected static class TestRollingUpgradeProcessor extends RollingUpgradeProcessor { + /** */ + public static CountDownLatch nodeJoinUnblockedLatch; + + /** */ + public static CountDownLatch nodeJoinValidationCompletedLatch; + + /** */ + public TestRollingUpgradeProcessor(GridKernalContext ctx, IgniteComponentFeatures testCoreFeatures) { + super(ctx, testCoreFeatures); + + ctx.event().addLocalEventListener(new TestNodeValidationFailedEventListener(), EVT_NODE_VALIDATION_FAILED); + } + + /** {@inheritDoc} */ + @Override @Nullable public IgniteNodeValidationResult validateNode(ClusterNode joiningNode) { + IgniteNodeValidationResult res = super.validateNode(joiningNode); + + if (res != null) + return res; + + if (nodeJoinValidationCompletedLatch != null) + nodeJoinValidationCompletedLatch.countDown(); + + if (nodeJoinUnblockedLatch != null) { + try { + assertTrue(nodeJoinUnblockedLatch.await(5_000, MILLISECONDS)); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + + log.error(e.getMessage(), e); + + return new IgniteNodeValidationResult(joiningNode.id(), e.getMessage()); + } + } + + return null; + } + } + + /** */ + protected static class TestNodeValidationFailedEventListener implements GridLocalEventListener, HighPriorityListener { + /** */ + public static CountDownLatch nodeValidationFailedEventUnblockedLatch; + + /** {@inheritDoc} */ + @Override public void onEvent(Event evt) { + try { + if (nodeValidationFailedEventUnblockedLatch != null) + assertTrue(nodeValidationFailedEventUnblockedLatch.await(5_000, MILLISECONDS)); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + + log.error(e.getMessage(), e); + + throw new IgniteException(e); + } + } + + /** {@inheritDoc} */ + @Override public int order() { + return 0; + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/ClusterVersionsRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/ClusterVersionsRollingUpgradeTest.java index a8606708e6238..2edd1dd2da1ef 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/ClusterVersionsRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/ClusterVersionsRollingUpgradeTest.java @@ -17,30 +17,18 @@ package org.apache.ignite.internal.processors.rollingupgrade; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; import java.util.UUID; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; -import org.apache.ignite.Ignition; import org.apache.ignite.configuration.BinaryConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.IgnitionEx; -import org.apache.ignite.internal.processors.rollingupgrade.RollingUpgradeProcessor.RollingUpgradeState; -import org.apache.ignite.internal.processors.rollingupgrade.feature.AbstractRollingUpgradeTest; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeature; import org.apache.ignite.internal.util.distributed.FullMessage; import org.apache.ignite.internal.util.distributed.InitMessage; import org.apache.ignite.internal.util.distributed.SingleNodeMessage; -import org.apache.ignite.internal.util.lang.ConsumerX; -import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage; @@ -52,7 +40,6 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED; -import static org.apache.ignite.internal.IgniteVersionUtils.semanticVersion; import static org.apache.ignite.internal.TestRecordingCommunicationSpi.spi; import static org.apache.ignite.internal.processors.rollingupgrade.feature.TestIgniteReleaseFeatures_2_19_2.VER_2_19_2_ID_1_FEATURE; import static org.apache.ignite.internal.processors.security.NodeSecurityContextPropagationTest.discoveryRingMessageWorkerQueue; @@ -63,11 +50,11 @@ public class ClusterVersionsRollingUpgradeTest extends AbstractRollingUpgradeTest { /** */ private static final String NODE_IS_INCOMPATIBLE_WIH_RU_ERR = - "The joining node version is incompatible with the current state of the cluster version Rolling Upgrade being in progress"; + "The joining node is incompatible with the current state of the version Rolling Upgrade being in progress"; /** */ private static final String MULTIPLE_VER_IN_TOP_ERR = - "Cluster version finalization failed. The topology contains nodes running multiple different versions"; + "Cluster version finalization failed. The cluster contains nodes running different versions of one or more components"; /** */ @Test @@ -84,7 +71,7 @@ public void testVersionUpgradeDisabledNodeJoin() throws Exception { checkVersionUpgradeInactive(TEST_DEFAULT_VER); - String msg = "The joining node version differs from the version of the cluster"; + String msg = "One or more component versions on the joining node differ from the corresponding versions active in the cluster"; checkJoinFailed(3, "2.18.0", msg); checkJoinFailed(3, "2.19.1", msg); @@ -321,7 +308,9 @@ public void testUpgradeBetweenVersionsWithCherryPicks() throws Exception { ru(1).enableVersionUpgrade(); checkJoinFailed(3, "2.20.0", - "Rolling Upgrade is not available between the current cluster logical version and the joining node product version"); + "Ignite component Rolling Upgrade is not supported between the component version active in the cluster and " + + "the version running on the joining node" + ); forAllNodes(nodeIdx -> upgradeNodeVersion(nodeIdx, "2.19.3", "2.20.1")); @@ -967,236 +956,4 @@ private boolean queuedDiscoveryMessageCountMatches(IgniteEx ignite, int expCnt, return expCnt == cnt; } - - /** */ - private void startCluster() throws Exception { - startCluster(TEST_DEFAULT_VER); - } - - /** */ - private void startCluster(String ver) throws Exception { - startGrid(0, ver); - startGrid(1, ver); - startClientGrid(2, ver); - } - - /** */ - private UUID extractActiveFinalizeProcessId(int nodeIdx) { - Object enable = U.field(grid(nodeIdx).context().rollingUpgrade(), "finalizeProc"); - - return U.field(enable, "locInitOpId"); - } - - /** */ - private void checkVersionFinalizationFailed(int initiatorNodeIdx, String msg) { - GridTestUtils.assertThrowsAnyCause( - log, - () -> { - ru(initiatorNodeIdx).finalizeClusterVersion(); - - return null; - }, - IgniteCheckedException.class, - msg - ); - } - - /** */ - private void checkJoinSuccess(int nodeIdx, boolean isClient) throws Exception { - checkJoinSuccess(nodeIdx, TEST_DEFAULT_VER, isClient); - } - - /** */ - private void checkJoinSuccess(int nodeIdx, String ver, boolean isClient) throws Exception { - int clusterSize = clusterNode().cluster().nodes().size(); - - startGrid(nodeIdx, ver, isClient); - - assertEquals(clusterSize + 1, clusterNode().cluster().nodes().size()); - } - - /** */ - private void checkJoinFailed(int nodeIdx, String ver, String msg) { - int expClusterSize = clusterNode().cluster().nodes().size(); - - GridTestUtils.assertThrowsAnyCause(log, () -> startGrid(nodeIdx, ver ), IgniteSpiException.class, msg); - GridTestUtils.assertThrowsAnyCause(log, () -> startClientGrid(nodeIdx, ver), IgniteSpiException.class, msg); - - assertEquals(expClusterSize, clusterNode().cluster().nodes().size()); - } - - /** */ - private IgniteEx clusterNode() { - return (IgniteEx)Ignition.allGrids().stream().filter(i -> !i.cluster().localNode().isClient()).findFirst().orElseThrow(); - } - - /** */ - private void checkVersionUpgradeInProgress(String srcVer, String targetVer) throws Exception { - checkVersionUpgradeInProgress(srcVer, srcVer, targetVer); - } - - /** */ - private void checkVersionUpgradeInProgress(String logicalVer, String srcVer, String targetVer) throws Exception { - checkVersionUpgradeEnabledStatus(true); - checkRollingUpgradeState(logicalVer, srcVer, targetVer); - - checkProductFeaturesActive(logicalVer); - - if (targetVer != null) - checkFeaturesNotActive(newFeatures(logicalVer, targetVer)); - } - - /** */ - private void checkVersionUpgradeInactive(String expVer) throws Exception { - checkVersionUpgradeEnabledStatus(false); - checkProductFeaturesActive(expVer); - } - - /** */ - private void checkVersionUpgradeEnabledStatus(boolean enabled) { - List cluster = Ignition.allGrids(); - - for (Ignite ignite : cluster) - assertEquals(enabled, ru(ignite).isVersionUpgradeEnabled()); - } - - /** */ - private void checkProductFeaturesActive(String ver) throws Exception { - Collection features = readDeclaredFeatures(ver); - - List cluster = Ignition.allGrids(); - - AtomicInteger validatedFeatures = new AtomicInteger(); - - for (Ignite ignite : cluster) { - for (IgniteFeature feature : features) { - assertTrue(isFeatureActive(ignite, feature)); - listenFeatureActivation(ignite, feature, validatedFeatures::incrementAndGet); - } - } - - assertEquals(cluster.size() * features.size(), validatedFeatures.get()); - } - - /** */ - private void checkFeaturesNotActive(Collection features) { - assertFalse(F.isEmpty(features)); - - for (Ignite ignite : Ignition.allGrids()) { - for (IgniteFeature feature : features) - assertFalse(isFeatureActive(ignite, feature)); - } - } - - /** */ - private Collection newFeatures(String srcVer, String targetVer) throws Exception { - Collection srcFeatures = readDeclaredFeatures(srcVer); - Collection targetFeatures = readDeclaredFeatures(targetVer); - - List res = new ArrayList<>(); - - for (IgniteFeature targetFeature : targetFeatures) { - if (!srcFeatures.contains(targetFeature)) - res.add(targetFeature); - } - - return res; - } - - /** */ - private void checkFeatureActivationSubscription(int nodeIdx, IgniteFeature feature, CountDownLatch featureActivationLatch) { - long expCnt = featureActivationLatch.getCount(); - - assertFalse(isFeatureActive(grid(nodeIdx), feature)); - listenFeatureActivation(grid(nodeIdx), feature, featureActivationLatch::countDown); - assertEquals(expCnt, featureActivationLatch.getCount()); - } - - /** */ - private void upgradeNodeVersion(int nodeIdx, String targetVer) throws Exception { - upgradeNodeVersion(nodeIdx, TEST_DEFAULT_VER, targetVer); - } - - /** */ - private void upgradeNodeVersion(int nodeIdx, String srcVer, String targetVer) throws Exception { - upgradeNodeVersion(nodeIdx, srcVer, srcVer, targetVer); - } - - /** */ - private void upgradeNodeVersion(int nodeIdx, String logicalVer, String srcVer, String targetVer) throws Exception { - boolean isClient = grid(nodeIdx).context().clientNode(); - - stopGrid(nodeIdx); - checkJoinSuccess(nodeIdx, targetVer, isClient); - checkVersionUpgradeInProgress(logicalVer, srcVer, targetVer); - } - - /** */ - private void finalizeClusterVersion(int nodeIdx, String expVer) throws Exception { - ru(nodeIdx).finalizeClusterVersion(); - - checkVersionUpgradeInactive(expVer); - } - - /** */ - private void restartNode(int nodeIdx) throws Exception { - String ver = semanticVersion(grid(nodeIdx).context().discovery().localNode().version()); - boolean isClient = grid(nodeIdx).context().clientNode(); - - stopGrid(nodeIdx); - checkJoinSuccess(nodeIdx, ver, isClient); - } - - /** */ - private void forAllNodes(ConsumerX nodeProcessor) throws Exception { - for (Ignite ignite : Ignition.allGrids()) - nodeProcessor.accept(getTestIgniteInstanceIndex(ignite.name())); - } - - /** */ - private void checkUpgradeFailed(int nodeIdx, String targetVer, String errMsg) throws Exception { - String srcVer = semanticVersion(grid(nodeIdx).context().discovery().localNode().version()); - boolean isClient = grid(nodeIdx).context().clientNode(); - - stopGrid(nodeIdx); - - GridTestUtils.assertThrowsAnyCause(log, () -> startGrid(nodeIdx, targetVer, isClient), IgniteSpiException.class, errMsg); - - startGrid(nodeIdx, srcVer, isClient); - } - - /** */ - private void checkRollingUpgradeState(String expLogicalVer, String expSrcVer, String expTargetVer) { - for (Ignite ignite : Ignition.allGrids()) { - assertTrue(ru(ignite).isVersionUpgradeEnabled()); - assertEquals(expLogicalVer, semanticVersion(ru(ignite).features().activeFeatures().version())); - - RollingUpgradeState state = ru(ignite).state(); - - assertEquals(expSrcVer, semanticVersion(state.srcVer)); - assertEquals(expTargetVer, state.targetVer == null ? null : semanticVersion(state.targetVer)); - } - } - - /** */ - private RollingUpgradeProcessor ru(int nodeIdx) { - return ru(grid(nodeIdx)); - } - - /** */ - private static RollingUpgradeProcessor ru(Ignite ignite) { - return ((IgniteEx)ignite).context().rollingUpgrade(); - } - - /** */ - private int coordinatorIndex() { - for (Ignite grid : IgnitionEx.allGridsx()) { - if (U.isLocalNodeCoordinator(((IgniteEx)grid).context().discovery())) - return getTestIgniteInstanceIndex(grid.name()); - } - - fail("Failed to resolve coordinator node"); - - return -1; - } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PluginVersionRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PluginVersionRollingUpgradeTest.java new file mode 100644 index 0000000000000..ab1a397c41ffa --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PluginVersionRollingUpgradeTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rollingupgrade; + +import org.junit.Test; + +/** */ +public class PluginVersionRollingUpgradeTest extends AbstractRollingUpgradeTest { + /** */ + @Test + public void testUpgradeDisabledJoinWithExtraComponent() throws Exception { + startGrid(0, "2.19.0"); + + String errMsg = "One or more component versions on the joining node differ from the corresponding versions active in the cluster"; + + checkJoinFailed(1, "2.19.0|1.0.0", errMsg); + } + + /** */ + @Test + public void testUpgradeDisabledJoinWithMissingComponent() throws Exception { + startGrid(0, "2.19.0 | 1.0.0"); + + checkJoinFailed(1, "2.19.0", false, + "One or more component versions on the joining node differ from the corresponding versions active in the cluster"); + + checkJoinFailed(1, "2.19.0 | 2.0.0", + "One or more component versions on the joining node differ from the corresponding versions active in the cluster"); + + checkJoinSuccess(1, "2.19.0", true); + } + + /** */ + @Test + public void testNewPluginActivation() throws Exception { + startCluster("2.19.0"); + + ru(1).enableVersionUpgrade(); + + checkVersionUpgradeInProgress("2.19.0 | null", "null | null"); + + forAllNodes(nodeIdx -> upgradeNodeVersion(nodeIdx, "2.19.0 | null", "2.19.0 | 1.0.0")); + + finalizeClusterVersion(1, "2.19.0 | 1.0.0"); + } + + /** */ + @Test + public void testPluginVersionUpgrade() throws Exception { + startCluster("2.19.0 | 1.0.0"); + + ru(1).enableVersionUpgrade(); + + checkVersionUpgradeInProgress("2.19.0 | 1.0.0", "null | null"); + + forAllNodes(nodeIdx -> upgradeNodeVersion(nodeIdx, "2.19.0 | 1.0.0", "2.19.0 | 2.0.0")); + + finalizeClusterVersion(1, "2.19.0 | 2.0.0"); + + ru(1).enableVersionUpgrade(); + + checkVersionUpgradeInProgress("2.19.0 | 2.0.0", "null | null"); + + forAllNodes(nodeIdx -> upgradeNodeVersion(nodeIdx, "2.19.0 | 2.0.0", "2.19.0 | 3.0.0")); + + finalizeClusterVersion(1, "2.19.0 | 3.0.0"); + } + + /** */ + @Test + public void testPluginAndIgniteVersionUpgrade() throws Exception { + startCluster("2.19.0 | 1.0.0"); + + ru(1).enableVersionUpgrade(); + + forAllNodes(nodeIdx -> upgradeNodeVersion(nodeIdx, "2.19.0 | 1.0.0", "2.20.0 | 2.0.0")); + + finalizeClusterVersion(1, "2.20.0 | 2.0.0"); + } + + /** */ + @Test + public void testComponentVersionValidationDuringFinalization() throws Exception { + startGrid(0, "2.19.0 | 1.0.0"); + startGrid(1, "2.19.0 | 1.0.0"); + startClientGrid(2, "2.19.0"); + + ru(1).enableVersionUpgrade(); + + startClientGrid(3, "2.19.0 | 2.0.0"); + checkVersionFinalizationFailed(0, "Cluster version finalization failed. The cluster contains nodes running " + + "different versions of one or more components"); + stopGrid(3); + + startGrid(4, "2.19.0 | 2.0.0"); + checkVersionFinalizationFailed(0, "Cluster version finalization failed. The cluster contains nodes running " + + "different versions of one or more components"); + stopGrid(4); + + startClientGrid(3, "2.19.0 | 1.0.0"); + startGrid(4, "2.19.0 | 1.0.0"); + + finalizeClusterVersion(1, "2.19.0 | 1.0.0"); + } + + /** */ + @Test + public void testNodeValidationDuringUpgrade() throws Exception { + startCluster("2.19.0 | 1.0.0"); + + ru(1).enableVersionUpgrade(); + + checkJoinFailed(3, "2.20.0", false, "Some components active in the cluster are not configured on the joining server node"); + checkJoinSuccess(3, "2.20.0", true); + + checkJoinFailed(4, "2.20.0 | 3.0.0", + "Ignite component Rolling Upgrade is not supported between the component version active in the cluster and " + + "the version running on the joining node" + ); + + checkJoinSuccess(4, "2.20.0 | 2.0.0", true); + + checkJoinFailed(5, "2.20.0 | 3.0.0", + "The joining node is incompatible with the current state of the version Rolling Upgrade being " + + "in progress." + ); + + upgradeNodeVersion(0, "2.19.0 | 1.0.0", "2.20.0 | 2.0.0"); + upgradeNodeVersion(1, "2.19.0 | 1.0.0", "2.20.0 | 2.0.0"); + upgradeNodeVersion(2, "2.19.0 | 1.0.0", "2.20.0 | 2.0.0"); + upgradeNodeVersion(4, "2.19.0 | 1.0.0", "2.20.0 | 2.0.0"); + + finalizeClusterVersion(1, "2.20.0 | 2.0.0"); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/AbstractRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/AbstractRollingUpgradeTest.java deleted file mode 100644 index ef4da447e7d61..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/AbstractRollingUpgradeTest.java +++ /dev/null @@ -1,265 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.rollingupgrade.feature; - -import java.util.Collection; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteException; -import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.events.Event; -import org.apache.ignite.internal.GridKernalContext; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.TestRecordingCommunicationSpi; -import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener; -import org.apache.ignite.internal.managers.eventstorage.HighPriorityListener; -import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor; -import org.apache.ignite.internal.processors.rollingupgrade.RollingUpgradeProcessor; -import org.apache.ignite.lang.IgniteProductVersion; -import org.apache.ignite.lang.IgniteRunnable; -import org.apache.ignite.plugin.AbstractTestPluginProvider; -import org.apache.ignite.plugin.PluginContext; -import org.apache.ignite.spi.IgniteNodeValidationResult; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.jspecify.annotations.Nullable; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.ignite.events.EventType.EVT_NODE_VALIDATION_FAILED; - -/** - * Provides the ability to override a node's version and supported {@link IgniteFeature}s in order to - * simulate a Rolling Upgrade procedure. - * - *

For testing purposes, the following "fake" Ignite versions and their corresponding - * {@link IgniteFeature}s have been introduced. These versions are used solely for testing and do not - * correspond to any actual Ignite releases. - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
VersionFeatures
2.18.0not supported
2.19.0{@code IgniteFeatureSet [0]}
2.19.1{@code IgniteFeatureSet [0, 1]}
2.19.2{@code IgniteFeatureSet [0 -> 2]}
2.19.3{@code IgniteFeatureSet [0 -> 2, 6]}
2.20.0{@code IgniteFeatureSet [0 -> 4]}
2.20.1{@code IgniteFeatureSet [0 -> 4, 6]}
2.21.0{@code IgniteFeatureSet [3 -> 6]}
2.21.1{@code IgniteFeatureSet [3 -> 7]}
- */ -public abstract class AbstractRollingUpgradeTest extends GridCommonAbstractTest { - /** */ - protected static final String TEST_DEFAULT_VER = "2.19.0"; - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - return getConfiguration(igniteInstanceName, TEST_DEFAULT_VER); - } - - /** */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); - - TestRollingUpgradeProcessor.nodeJoinValidationCompletedLatch = null; - TestRollingUpgradeProcessor.nodeJoinUnblockedLatch = null; - TestNodeValidationFailedEventListener.nodeValidationFailedEventUnblockedLatch = null; - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - super.afterTest(); - - stopAllGrids(); - } - - /** */ - protected IgniteConfiguration getConfiguration(int idx, String ver) throws Exception { - return getConfiguration(getTestIgniteInstanceName(idx), ver); - } - - /** */ - protected IgniteConfiguration getConfiguration(String igniteInstanceName, String ver) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - cfg.setCommunicationSpi(new TestRecordingCommunicationSpi()); - - IgniteProductFeatures testNodeVerFeatures = new IgniteProductFeatures( - IgniteProductVersion.fromString(ver), - IgniteFeatureSet.buildFrom(readDeclaredFeatures(ver))); - - cfg.setPluginProviders(new AbstractTestPluginProvider() { - @Override public String name() { - return "test-rolling-upgrade-processor-provider"; - } - - /** {@inheritDoc} */ - @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { - if (cls.isAssignableFrom(DiscoveryNodeValidationProcessor.class)) - return (T)new TestRollingUpgradeProcessor(((IgniteEx)ctx.grid()).context(), testNodeVerFeatures); - - return null; - } - }); - - TcpDiscoverySpi discoSpi = new TcpDiscoverySpi() { - @Override public void setNodeAttributes(Map attrs, IgniteProductVersion ignored) { - super.setNodeAttributes(attrs, IgniteProductVersion.fromString(ver)); - } - }; - - discoSpi.setIpFinder(((TcpDiscoverySpi)cfg.getDiscoverySpi()).getIpFinder()); - - cfg.setDiscoverySpi(discoSpi); - - return cfg; - } - - /** */ - protected IgniteEx startGrid(int idx, String ver) throws Exception { - return startGrid(getConfiguration(idx, ver)); - } - - /** */ - protected IgniteEx startClientGrid(int idx, String ver) throws Exception { - return startClientGrid(getConfiguration(idx, ver)); - } - - /** */ - protected IgniteEx startGrid(int idx, String ver, boolean isClient) throws Exception { - return isClient ? startClientGrid(idx, ver) : startGrid(idx, ver); - } - - /** */ - protected Collection readDeclaredFeatures(String ver) throws Exception { - Class cls = Class.forName( - TestIgniteReleaseFeatures_2_18_0.class.getPackageName() + ".TestIgniteReleaseFeatures_" + ver.replace(".", "_")); - - return IgniteFeatureSet.readDeclaredFeatures(cls); - } - - /** */ - protected static class TestRollingUpgradeProcessor extends RollingUpgradeProcessor { - /** */ - public static CountDownLatch nodeJoinUnblockedLatch; - - /** */ - public static CountDownLatch nodeJoinValidationCompletedLatch; - - /** */ - public TestRollingUpgradeProcessor(GridKernalContext ctx, IgniteProductFeatures testNodeVerFeatures) { - super(ctx, testNodeVerFeatures); - - ctx.event().addLocalEventListener(new TestNodeValidationFailedEventListener(), EVT_NODE_VALIDATION_FAILED); - } - - /** {@inheritDoc} */ - @Override @Nullable public IgniteNodeValidationResult validateNode(ClusterNode joiningNode) { - IgniteNodeValidationResult res = super.validateNode(joiningNode); - - if (res != null) - return res; - - if (nodeJoinValidationCompletedLatch != null) - nodeJoinValidationCompletedLatch.countDown(); - - if (nodeJoinUnblockedLatch != null) { - try { - assertTrue(nodeJoinUnblockedLatch.await(5_000, MILLISECONDS)); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - - log.error(e.getMessage(), e); - - return new IgniteNodeValidationResult(joiningNode.id(), e.getMessage()); - } - } - - return null; - } - } - - /** */ - protected boolean isFeatureActive(Ignite ignite, IgniteFeature feature) { - return ((IgniteEx)ignite).context().rollingUpgrade().features().isActive(feature); - } - - /** */ - protected void listenFeatureActivation(Ignite ignite, IgniteFeature feature, IgniteRunnable lsnr) { - ((IgniteEx)ignite).context().rollingUpgrade().features().listenActivation(feature, lsnr); - } - - /** */ - protected static class TestNodeValidationFailedEventListener implements GridLocalEventListener, HighPriorityListener { - /** */ - public static CountDownLatch nodeValidationFailedEventUnblockedLatch; - - /** {@inheritDoc} */ - @Override public void onEvent(Event evt) { - try { - if (nodeValidationFailedEventUnblockedLatch != null) - assertTrue(nodeValidationFailedEventUnblockedLatch.await(5_000, MILLISECONDS)); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - - log.error(e.getMessage(), e); - - throw new IgniteException(e); - } - } - - /** {@inheritDoc} */ - @Override public int order() { - return 0; - } - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginComponentFeaturesProvider.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginComponentFeaturesProvider.java new file mode 100644 index 0000000000000..4273558da2004 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginComponentFeaturesProvider.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rollingupgrade.feature; + +import java.util.Collection; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.processors.rollingupgrade.AbstractRollingUpgradeTest; +import org.apache.ignite.lang.IgniteProductVersion; + +/** */ +public class TestPluginComponentFeaturesProvider implements IgniteComponentFeaturesProvider { + /** */ + private final String pluginVer; + + /** */ + public TestPluginComponentFeaturesProvider(String pluginVer) { + this.pluginVer = pluginVer; + } + + /** {@inheritDoc} */ + @Override public String componentName() { + return TestPluginFeature.COMPONENT_NAME; + } + + /** {@inheritDoc} */ + @Override public IgniteProductVersion componentVersion() { + return IgniteProductVersion.fromString(pluginVer); + } + + /** {@inheritDoc} */ + @Override public Collection features() { + try { + return AbstractRollingUpgradeTest.readDeclaredPluginFeatures(pluginVer); + } + catch (Exception e) { + throw new IgniteException(e); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginFeature.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginFeature.java new file mode 100644 index 0000000000000..c7527529ca923 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginFeature.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rollingupgrade.feature; + +/** */ +public class TestPluginFeature implements IgniteFeature { + /** */ + public static final String COMPONENT_NAME = "plugin"; + + /** */ + private final int id; + + /** */ + public TestPluginFeature(int id) { + this.id = id; + } + + /** {@inheritDoc} */ + @Override public int id() { + return id; + } + + /** {@inheritDoc} */ + @Override public String componentName() { + return COMPONENT_NAME; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginReleaseFeatures_1_0_0.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginReleaseFeatures_1_0_0.java new file mode 100644 index 0000000000000..65d0521b2e1e3 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginReleaseFeatures_1_0_0.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rollingupgrade.feature; + +/** */ +public class TestPluginReleaseFeatures_1_0_0 { + /** */ + public static final IgniteFeature VER_1_0_0_ID_0_FEATURE = new TestPluginFeature(0); +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginReleaseFeatures_2_0_0.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginReleaseFeatures_2_0_0.java new file mode 100644 index 0000000000000..d661f5cff66bc --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginReleaseFeatures_2_0_0.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rollingupgrade.feature; + +/** */ +public class TestPluginReleaseFeatures_2_0_0 { + /** */ + public static final IgniteFeature VER_2_0_0_ID_1_FEATURE = new TestPluginFeature(1); +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginReleaseFeatures_3_0_0.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginReleaseFeatures_3_0_0.java new file mode 100644 index 0000000000000..b40cea1590d2b --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestPluginReleaseFeatures_3_0_0.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rollingupgrade.feature; + +/** */ +public class TestPluginReleaseFeatures_3_0_0 { + /** */ + public static final IgniteFeature VER_3_0_0_ID_2_FEATURE = new TestPluginFeature(2); +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java index f4d278c1c8c11..73e933025806b 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java @@ -64,6 +64,7 @@ import org.apache.ignite.internal.processors.odbc.OdbcEscapeSequenceSelfTest; import org.apache.ignite.internal.processors.odbc.SqlListenerUtilsTest; import org.apache.ignite.internal.processors.rollingupgrade.ClusterVersionsRollingUpgradeTest; +import org.apache.ignite.internal.processors.rollingupgrade.PluginVersionRollingUpgradeTest; import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureSetTest; import org.apache.ignite.internal.product.GridProductVersionSelfTest; import org.apache.ignite.internal.util.nio.IgniteExceptionInNioWorkerSelfTest; @@ -106,6 +107,7 @@ GridMessagingNoPeerClassLoadingSelfTest.class, ClusterVersionsRollingUpgradeTest.class, + PluginVersionRollingUpgradeTest.class, GridProductVersionSelfTest.class, GridAffinityAssignmentV2Test.class, GridAffinityAssignmentV2TestNoOptimizations.class,